parent
eaf607e88b
commit
ceb51ddaa1
@ -22,7 +22,7 @@
|
|||||||
var StringDecoder = exports.StringDecoder = function(encoding) {
|
var StringDecoder = exports.StringDecoder = function(encoding) {
|
||||||
this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, '');
|
this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, '');
|
||||||
if (this.encoding === 'utf8') {
|
if (this.encoding === 'utf8') {
|
||||||
this.charBuffer = new Buffer(4);
|
this.charBuffer = new Buffer(6);
|
||||||
this.charReceived = 0;
|
this.charReceived = 0;
|
||||||
this.charLength = 0;
|
this.charLength = 0;
|
||||||
}
|
}
|
||||||
@ -36,16 +36,18 @@ StringDecoder.prototype.write = function(buffer) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var charStr = '';
|
var charStr = '';
|
||||||
|
var offset = 0;
|
||||||
// if our last write ended with an incomplete multibyte character
|
// if our last write ended with an incomplete multibyte character
|
||||||
if (this.charLength) {
|
while (this.charLength) {
|
||||||
// determine how many remaining bytes this buffer has to offer for this char
|
// determine how many remaining bytes this buffer has to offer for this char
|
||||||
var i = (buffer.length >= this.charLength - this.charReceived) ?
|
var i = (buffer.length >= this.charLength - this.charReceived) ?
|
||||||
this.charLength - this.charReceived :
|
this.charLength - this.charReceived :
|
||||||
buffer.length;
|
buffer.length;
|
||||||
|
|
||||||
// add the new bytes to the char buffer
|
// add the new bytes to the char buffer
|
||||||
buffer.copy(this.charBuffer, this.charReceived, 0, i);
|
buffer.copy(this.charBuffer, this.charReceived, offset, i);
|
||||||
this.charReceived += i;
|
this.charReceived += (i - offset);
|
||||||
|
offset = i;
|
||||||
|
|
||||||
if (this.charReceived < this.charLength) {
|
if (this.charReceived < this.charLength) {
|
||||||
// still not enough chars in this buffer? wait for more ...
|
// still not enough chars in this buffer? wait for more ...
|
||||||
@ -54,6 +56,16 @@ StringDecoder.prototype.write = function(buffer) {
|
|||||||
|
|
||||||
// get the character that was split
|
// get the character that was split
|
||||||
charStr = this.charBuffer.slice(0, this.charLength).toString();
|
charStr = this.charBuffer.slice(0, this.charLength).toString();
|
||||||
|
|
||||||
|
// lead surrogate (D800-DBFF) is also the incomplete character
|
||||||
|
if (this.charLength === 3) {
|
||||||
|
var charCode = charStr.charCodeAt(0);
|
||||||
|
if (charCode >= 0xD800 && charCode <= 0xDBFF) {
|
||||||
|
charStr = '';
|
||||||
|
this.charLength += 3; // size of trail surrogate (DC00-DFFF)
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
this.charReceived = this.charLength = 0;
|
this.charReceived = this.charLength = 0;
|
||||||
|
|
||||||
// if there are no more bytes in this buffer, just emit our char
|
// if there are no more bytes in this buffer, just emit our char
|
||||||
@ -61,6 +73,7 @@ StringDecoder.prototype.write = function(buffer) {
|
|||||||
|
|
||||||
// otherwise cut off the characters end from the beginning of this buffer
|
// otherwise cut off the characters end from the beginning of this buffer
|
||||||
buffer = buffer.slice(i, buffer.length);
|
buffer = buffer.slice(i, buffer.length);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -93,18 +106,26 @@ StringDecoder.prototype.write = function(buffer) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!this.charLength) {
|
var end = buffer.length;
|
||||||
// no incomplete char at the end of this buffer, emit the whole thing
|
if (this.charLength) {
|
||||||
return charStr + buffer.toString();
|
// buffer the incomplete character bytes we got
|
||||||
|
buffer.copy(this.charBuffer, 0, buffer.length - i, buffer.length);
|
||||||
|
this.charReceived = i;
|
||||||
|
end -= i;
|
||||||
}
|
}
|
||||||
|
|
||||||
// buffer the incomplete character bytes we got
|
charStr += buffer.toString('utf8', 0, end);
|
||||||
buffer.copy(this.charBuffer, 0, buffer.length - i, buffer.length);
|
|
||||||
this.charReceived = i;
|
|
||||||
|
|
||||||
if (buffer.length - i > 0) {
|
// lead surrogate (D800-DBFF) is also the incomplete character
|
||||||
// buffer had more bytes before the incomplete char, emit them
|
end = charStr.length - 1;
|
||||||
return charStr + buffer.toString('utf8', 0, buffer.length - i);
|
var charCode = charStr.charCodeAt(end);
|
||||||
|
if (charCode >= 0xD800 && charCode <= 0xDBFF) {
|
||||||
|
// CESU-8 represents each of Surrogate Pair by 3-bytes
|
||||||
|
this.charLength += 3
|
||||||
|
this.charReceived += 3
|
||||||
|
this.charBuffer.copy(this.charBuffer, 3, 0, 3);
|
||||||
|
this.charBuffer.write(charStr.charAt(end));
|
||||||
|
return charStr.substring(0, end);
|
||||||
}
|
}
|
||||||
|
|
||||||
// or just emit the charStr
|
// or just emit the charStr
|
||||||
|
@ -46,6 +46,49 @@ s += decoder.write(buffer.slice(2, 3));
|
|||||||
s += decoder.write(buffer.slice(3, 4));
|
s += decoder.write(buffer.slice(3, 4));
|
||||||
assert.ok(s.length > 0);
|
assert.ok(s.length > 0);
|
||||||
|
|
||||||
|
// CESU-8
|
||||||
|
buffer = new Buffer('EDA0BDEDB18D', 'hex'); // THUMBS UP SIGN (in CESU-8)
|
||||||
|
var s = '';
|
||||||
|
s += decoder.write(buffer.slice(0, 1));
|
||||||
|
s += decoder.write(buffer.slice(1, 2));
|
||||||
|
s += decoder.write(buffer.slice(2, 3)); // complete lead surrogate
|
||||||
|
assert.equal(s, '');
|
||||||
|
s += decoder.write(buffer.slice(3, 4));
|
||||||
|
s += decoder.write(buffer.slice(4, 5));
|
||||||
|
s += decoder.write(buffer.slice(5, 6)); // complete trail surrogate
|
||||||
|
assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
|
||||||
|
|
||||||
|
var s = '';
|
||||||
|
s += decoder.write(buffer.slice(0, 2));
|
||||||
|
s += decoder.write(buffer.slice(2, 4)); // complete lead surrogate
|
||||||
|
assert.equal(s, '');
|
||||||
|
s += decoder.write(buffer.slice(4, 6)); // complete trail surrogate
|
||||||
|
assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
|
||||||
|
|
||||||
|
var s = '';
|
||||||
|
s += decoder.write(buffer.slice(0, 3)); // complete lead surrogate
|
||||||
|
assert.equal(s, '');
|
||||||
|
s += decoder.write(buffer.slice(3, 6)); // complete trail surrogate
|
||||||
|
assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
|
||||||
|
|
||||||
|
var s = '';
|
||||||
|
s += decoder.write(buffer.slice(0, 4)); // complete lead surrogate
|
||||||
|
assert.equal(s, '');
|
||||||
|
s += decoder.write(buffer.slice(4, 5));
|
||||||
|
s += decoder.write(buffer.slice(5, 6)); // complete trail surrogate
|
||||||
|
assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
|
||||||
|
|
||||||
|
var s = '';
|
||||||
|
s += decoder.write(buffer.slice(0, 5)); // complete lead surrogate
|
||||||
|
assert.equal(s, '');
|
||||||
|
s += decoder.write(buffer.slice(5, 6)); // complete trail surrogate
|
||||||
|
assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
|
||||||
|
|
||||||
|
var s = '';
|
||||||
|
s += decoder.write(buffer.slice(0, 6));
|
||||||
|
assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
|
||||||
|
|
||||||
|
|
||||||
// A mixed ascii and non-ascii string
|
// A mixed ascii and non-ascii string
|
||||||
// Test stolen from deps/v8/test/cctest/test-strings.cc
|
// Test stolen from deps/v8/test/cctest/test-strings.cc
|
||||||
// U+02E4 -> CB A4
|
// U+02E4 -> CB A4
|
||||||
|
Loading…
x
Reference in New Issue
Block a user