test: remove duplicate encoding tests in favor of WPT
PR-URL: https://github.com/nodejs/node/pull/25321 Reviewed-By: Rich Trott <rtrott@gmail.com> Reviewed-By: James M Snell <jasnell@gmail.com> Reviewed-By: Ruben Bridgewater <ruben@bridgewater.de>
This commit is contained in:
Joyee Cheung
parent
16dfa0f309
commit
dce2f3ed93
@ -1,74 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
// From: https://github.com/w3c/web-platform-tests/blob/master/encoding/api-basics.html
|
||||
// TODO(joyeecheung): replace this with WPT
|
||||
|
||||
const common = require('../common');
|
||||
|
||||
if (!common.hasIntl)
|
||||
common.skip('missing Intl');
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
function testDecodeSample(encoding, string, bytes) {
|
||||
assert.strictEqual(
|
||||
new TextDecoder(encoding).decode(new Uint8Array(bytes)),
|
||||
string);
|
||||
assert.strictEqual(
|
||||
new TextDecoder(encoding).decode(new Uint8Array(bytes).buffer),
|
||||
string);
|
||||
}
|
||||
|
||||
// `z` (ASCII U+007A), cent (Latin-1 U+00A2), CJK water (BMP U+6C34),
|
||||
// G-Clef (non-BMP U+1D11E), PUA (BMP U+F8FF), PUA (non-BMP U+10FFFD)
|
||||
// byte-swapped BOM (non-character U+FFFE)
|
||||
const sample = 'z\xA2\u6C34\uD834\uDD1E\uF8FF\uDBFF\uDFFD\uFFFE';
|
||||
|
||||
{
|
||||
const encoding = 'utf-8';
|
||||
const string = sample;
|
||||
const bytes = [
|
||||
0x7A, 0xC2, 0xA2, 0xE6, 0xB0, 0xB4,
|
||||
0xF0, 0x9D, 0x84, 0x9E, 0xEF, 0xA3,
|
||||
0xBF, 0xF4, 0x8F, 0xBF, 0xBD, 0xEF,
|
||||
0xBF, 0xBE
|
||||
];
|
||||
const encoded = new TextEncoder().encode(string);
|
||||
assert.deepStrictEqual([].slice.call(encoded), bytes);
|
||||
assert.strictEqual(
|
||||
new TextDecoder(encoding).decode(new Uint8Array(bytes)),
|
||||
string);
|
||||
assert.strictEqual(
|
||||
new TextDecoder(encoding).decode(new Uint8Array(bytes).buffer),
|
||||
string);
|
||||
}
|
||||
|
||||
testDecodeSample(
|
||||
'utf-16le',
|
||||
sample,
|
||||
[
|
||||
0x7A, 0x00, 0xA2, 0x00, 0x34, 0x6C,
|
||||
0x34, 0xD8, 0x1E, 0xDD, 0xFF, 0xF8,
|
||||
0xFF, 0xDB, 0xFD, 0xDF, 0xFE, 0xFF
|
||||
]
|
||||
);
|
||||
|
||||
testDecodeSample(
|
||||
'utf-16be',
|
||||
sample,
|
||||
[
|
||||
0x00, 0x7A, 0x00, 0xA2, 0x6C, 0x34,
|
||||
0xD8, 0x34, 0xDD, 0x1E, 0xF8, 0xFF,
|
||||
0xDB, 0xFF, 0xDF, 0xFD, 0xFF, 0xFE
|
||||
]
|
||||
);
|
||||
|
||||
testDecodeSample(
|
||||
'utf-16',
|
||||
sample,
|
||||
[
|
||||
0x7A, 0x00, 0xA2, 0x00, 0x34, 0x6C,
|
||||
0x34, 0xD8, 0x1E, 0xDD, 0xFF, 0xF8,
|
||||
0xFF, 0xDB, 0xFD, 0xDF, 0xFE, 0xFF
|
||||
]
|
||||
);
|
||||
@ -8,8 +8,6 @@ const common = require('../common');
|
||||
if (!common.hasIntl)
|
||||
common.skip('missing Intl');
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
{
|
||||
[
|
||||
{ encoding: 'utf-8', sequence: [0xC0] },
|
||||
@ -28,12 +26,6 @@ const assert = require('assert');
|
||||
`The encoded data was not valid for encoding ${testCase.encoding}`
|
||||
}
|
||||
);
|
||||
|
||||
// TODO(joyeecheung): remove this when WPT is ported
|
||||
assert.strictEqual(
|
||||
new TextDecoder(testCase.encoding).decode(data),
|
||||
'\uFFFD'
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@ -42,10 +34,6 @@ const assert = require('assert');
|
||||
const odd = new Uint8Array([0x00]);
|
||||
const even = new Uint8Array([0x00, 0x00]);
|
||||
|
||||
// TODO(joyeecheung): remove this when WPT is ported
|
||||
assert.strictEqual(decoder.decode(odd, { stream: true }), '');
|
||||
assert.strictEqual(decoder.decode(odd), '\u0000');
|
||||
|
||||
common.expectsError(
|
||||
() => {
|
||||
decoder.decode(even, { stream: true });
|
||||
@ -69,8 +57,4 @@ const assert = require('assert');
|
||||
'The encoded data was not valid for encoding utf-16le'
|
||||
}
|
||||
);
|
||||
|
||||
// TODO(joyeecheung): remove this when WPT is ported
|
||||
assert.strictEqual(decoder.decode(even, { stream: true }), '\u0000');
|
||||
assert.strictEqual(decoder.decode(even), '\u0000');
|
||||
}
|
||||
|
||||
@ -8,8 +8,6 @@ const common = require('../common');
|
||||
if (!common.hasIntl)
|
||||
common.skip('missing Intl');
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const bad = [
|
||||
{ encoding: 'utf-8', input: [0xFF], name: 'invalid code' },
|
||||
{ encoding: 'utf-8', input: [0xC0], name: 'ends early' },
|
||||
@ -82,11 +80,3 @@ bad.forEach((t) => {
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
// TODO(joyeecheung): remove this when WPT is ported
|
||||
{
|
||||
assert('fatal' in new TextDecoder());
|
||||
assert.strictEqual(typeof new TextDecoder().fatal, 'boolean');
|
||||
assert(!new TextDecoder().fatal);
|
||||
assert(new TextDecoder('utf-8', { fatal: true }).fatal);
|
||||
}
|
||||
|
||||
@ -8,8 +8,6 @@ const common = require('../common');
|
||||
if (!common.hasIntl)
|
||||
common.skip('missing Intl');
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const bad = [
|
||||
{
|
||||
encoding: 'utf-16le',
|
||||
@ -44,11 +42,6 @@ const bad = [
|
||||
];
|
||||
|
||||
bad.forEach((t) => {
|
||||
// TODO(joyeecheung): remove this when WPT is ported
|
||||
assert.strictEqual(
|
||||
new TextDecoder(t.encoding).decode(new Uint8Array(t.input)),
|
||||
t.expected);
|
||||
|
||||
common.expectsError(
|
||||
() => {
|
||||
new TextDecoder(t.encoding, { fatal: true })
|
||||
|
||||
@ -1,53 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
// From: https://github.com/w3c/web-platform-tests/blob/fa9436d12c/encoding/api-surrogates-utf8.html
|
||||
// TODO(joyeecheung): replace this with WPT
|
||||
|
||||
require('../common');
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const badStrings = [
|
||||
{
|
||||
input: 'abc123',
|
||||
expected: [0x61, 0x62, 0x63, 0x31, 0x32, 0x33],
|
||||
decoded: 'abc123',
|
||||
name: 'Sanity check'
|
||||
},
|
||||
{
|
||||
input: '\uD800',
|
||||
expected: [0xef, 0xbf, 0xbd],
|
||||
decoded: '\uFFFD',
|
||||
name: 'Surrogate half (low)'
|
||||
},
|
||||
{
|
||||
input: '\uDC00',
|
||||
expected: [0xef, 0xbf, 0xbd],
|
||||
decoded: '\uFFFD',
|
||||
name: 'Surrogate half (high)'
|
||||
},
|
||||
{
|
||||
input: 'abc\uD800123',
|
||||
expected: [0x61, 0x62, 0x63, 0xef, 0xbf, 0xbd, 0x31, 0x32, 0x33],
|
||||
decoded: 'abc\uFFFD123',
|
||||
name: 'Surrogate half (low), in a string'
|
||||
},
|
||||
{
|
||||
input: 'abc\uDC00123',
|
||||
expected: [0x61, 0x62, 0x63, 0xef, 0xbf, 0xbd, 0x31, 0x32, 0x33],
|
||||
decoded: 'abc\uFFFD123',
|
||||
name: 'Surrogate half (high), in a string'
|
||||
},
|
||||
{
|
||||
input: '\uDC00\uD800',
|
||||
expected: [0xef, 0xbf, 0xbd, 0xef, 0xbf, 0xbd],
|
||||
decoded: '\uFFFD\uFFFD',
|
||||
name: 'Wrong order'
|
||||
}
|
||||
];
|
||||
|
||||
badStrings.forEach((t) => {
|
||||
const encoded = new TextEncoder().encode(t.input);
|
||||
assert.deepStrictEqual([].slice.call(encoded), t.expected);
|
||||
assert.strictEqual(new TextDecoder('utf-8').decode(encoded), t.decoded);
|
||||
});
|
||||
@ -1,44 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
// From: https://github.com/w3c/web-platform-tests/blob/7f567fa29c/encoding/textdecoder-ignorebom.html
|
||||
// TODO(joyeecheung): replace this with WPT
|
||||
|
||||
const common = require('../common');
|
||||
|
||||
if (!common.hasIntl)
|
||||
common.skip('missing Intl');
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const cases = [
|
||||
{
|
||||
encoding: 'utf-8',
|
||||
bytes: [0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63]
|
||||
},
|
||||
{
|
||||
encoding: 'utf-16le',
|
||||
bytes: [0xFF, 0xFE, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00]
|
||||
},
|
||||
{
|
||||
encoding: 'utf-16be',
|
||||
bytes: [0xFE, 0xFF, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63]
|
||||
}
|
||||
];
|
||||
|
||||
cases.forEach((testCase) => {
|
||||
const BOM = '\uFEFF';
|
||||
let decoder = new TextDecoder(testCase.encoding, { ignoreBOM: true });
|
||||
const bytes = new Uint8Array(testCase.bytes);
|
||||
assert.strictEqual(decoder.decode(bytes), `${BOM}abc`);
|
||||
decoder = new TextDecoder(testCase.encoding, { ignoreBOM: false });
|
||||
assert.strictEqual(decoder.decode(bytes), 'abc');
|
||||
decoder = new TextDecoder(testCase.encoding);
|
||||
assert.strictEqual(decoder.decode(bytes), 'abc');
|
||||
});
|
||||
|
||||
{
|
||||
assert('ignoreBOM' in new TextDecoder());
|
||||
assert.strictEqual(typeof new TextDecoder().ignoreBOM, 'boolean');
|
||||
assert(!new TextDecoder().ignoreBOM);
|
||||
assert(new TextDecoder('utf-8', { ignoreBOM: true }).ignoreBOM);
|
||||
}
|
||||
@ -1,50 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
// From: https://github.com/w3c/web-platform-tests/blob/fa9436d12c/encoding/textdecoder-streaming.html
|
||||
// TODO(joyeecheung): replace this with WPT
|
||||
|
||||
const common = require('../common');
|
||||
|
||||
if (!common.hasIntl)
|
||||
common.skip('missing Intl');
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const string =
|
||||
'\x00123ABCabc\x80\xFF\u0100\u1000\uFFFD\uD800\uDC00\uDBFF\uDFFF';
|
||||
const octets = {
|
||||
'utf-8': [
|
||||
0x00, 0x31, 0x32, 0x33, 0x41, 0x42, 0x43, 0x61, 0x62, 0x63, 0xc2, 0x80,
|
||||
0xc3, 0xbf, 0xc4, 0x80, 0xe1, 0x80, 0x80, 0xef, 0xbf, 0xbd, 0xf0, 0x90,
|
||||
0x80, 0x80, 0xf4, 0x8f, 0xbf, 0xbf],
|
||||
'utf-16le': [
|
||||
0x00, 0x00, 0x31, 0x00, 0x32, 0x00, 0x33, 0x00, 0x41, 0x00, 0x42, 0x00,
|
||||
0x43, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00, 0x80, 0x00, 0xFF, 0x00,
|
||||
0x00, 0x01, 0x00, 0x10, 0xFD, 0xFF, 0x00, 0xD8, 0x00, 0xDC, 0xFF, 0xDB,
|
||||
0xFF, 0xDF],
|
||||
'utf-16be': [
|
||||
0x00, 0x00, 0x00, 0x31, 0x00, 0x32, 0x00, 0x33, 0x00, 0x41, 0x00, 0x42,
|
||||
0x00, 0x43, 0x00, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00, 0x80, 0x00, 0xFF,
|
||||
0x01, 0x00, 0x10, 0x00, 0xFF, 0xFD, 0xD8, 0x00, 0xDC, 0x00, 0xDB, 0xFF,
|
||||
0xDF, 0xFF]
|
||||
};
|
||||
|
||||
Object.keys(octets).forEach((encoding) => {
|
||||
if (encoding === 'utf-16be' && !common.hasIntl) {
|
||||
console.log('skipping utf-16be because missing Intl');
|
||||
return;
|
||||
}
|
||||
for (let len = 1; len <= 5; ++len) {
|
||||
const encoded = octets[encoding];
|
||||
const decoder = new TextDecoder(encoding);
|
||||
let out = '';
|
||||
for (let i = 0; i < encoded.length; i += len) {
|
||||
const sub = [];
|
||||
for (let j = i; j < encoded.length && j < i + len; ++j)
|
||||
sub.push(encoded[j]);
|
||||
out += decoder.decode(new Uint8Array(sub), { stream: true });
|
||||
}
|
||||
out += decoder.decode();
|
||||
assert.strictEqual(out, string);
|
||||
}
|
||||
});
|
||||
@ -1,49 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
// From: https://github.com/w3c/web-platform-tests/blob/fa9436d12c/encoding/textencoder-utf16-surrogates.html
|
||||
// TODO(joyeecheung): replace this with WPT
|
||||
|
||||
require('../common');
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const bad = [
|
||||
{
|
||||
input: '\uD800',
|
||||
expected: '\uFFFD',
|
||||
name: 'lone surrogate lead'
|
||||
},
|
||||
{
|
||||
input: '\uDC00',
|
||||
expected: '\uFFFD',
|
||||
name: 'lone surrogate trail'
|
||||
},
|
||||
{
|
||||
input: '\uD800\u0000',
|
||||
expected: '\uFFFD\u0000',
|
||||
name: 'unmatched surrogate lead'
|
||||
},
|
||||
{
|
||||
input: '\uDC00\u0000',
|
||||
expected: '\uFFFD\u0000',
|
||||
name: 'unmatched surrogate trail'
|
||||
},
|
||||
{
|
||||
input: '\uDC00\uD800',
|
||||
expected: '\uFFFD\uFFFD',
|
||||
name: 'swapped surrogate pair'
|
||||
},
|
||||
{
|
||||
input: '\uD834\uDD1E',
|
||||
expected: '\uD834\uDD1E',
|
||||
name: 'properly encoded MUSICAL SYMBOL G CLEF (U+1D11E)'
|
||||
}
|
||||
];
|
||||
|
||||
bad.forEach((t) => {
|
||||
const encoded = new TextEncoder().encode(t.input);
|
||||
const decoded = new TextDecoder().decode(encoded);
|
||||
assert.strictEqual(decoded, t.expected);
|
||||
});
|
||||
|
||||
assert.strictEqual(new TextEncoder().encode().length, 0);
|
||||
Loading…
x
Reference in New Issue
Block a user