1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove get_actual_encoding() and the dynamic endian detection for dummy UTF-16/UTF-32

Browse Source 
* And simplify callers of get_actual_encoding().
* See [Feature #18949].
* See https://github.com/ruby/ruby/pull/6322#issuecomment-1242758474
This commit is contained in:
Benoit Daloze 2022-09-10 19:15:49 +02:00
parent ed029e9bd4
commit 6525b6f760
7 changed files with 26 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
enc/encdb.c
View File

@ -17,7 +17,7 @@
#define ENC_DEFINE(name) rb_encdb_declare(name) #define ENC_DEFINE(name) rb_encdb_declare(name)
#define ENC_SET_BASE(name, orig) rb_enc_set_base((name), (orig)) #define ENC_SET_BASE(name, orig) rb_enc_set_base((name), (orig))
#define ENC_SET_DUMMY(name, orig) rb_enc_set_dummy(name) #define ENC_SET_DUMMY(name, orig) rb_enc_set_dummy(name)
#define ENC_DUMMY_UNICODE(name) rb_encdb_set_unicode(rb_enc_set_dummy(ENC_REPLICATE((name), name "BE"))) #define ENC_DUMMY_UNICODE(name) ENC_DUMMY(name)
void void
Init_encdb(void) Init_encdb(void)

2
enc/utf_16_32.h
View File

@ -1,5 +1,5 @@
#include "regenc.h" #include "regenc.h"
/* dummy for unsupported, stateful encoding */ /* dummy for unsupported, stateful encoding */
#define ENC_DUMMY_UNICODE(name) ENC_REPLICATE(name, name "BE") #define ENC_DUMMY_UNICODE(name) ENC_DUMMY(name)
ENC_DUMMY_UNICODE("UTF-16"); ENC_DUMMY_UNICODE("UTF-16");
ENC_DUMMY_UNICODE("UTF-32"); ENC_DUMMY_UNICODE("UTF-32");

9
encoding.c
View File

@ -50,7 +50,6 @@ void rb_encdb_declare(const char *name);
int rb_encdb_replicate(const char *name, const char *orig); int rb_encdb_replicate(const char *name, const char *orig);
int rb_encdb_dummy(const char *name); int rb_encdb_dummy(const char *name);
int rb_encdb_alias(const char *alias, const char *orig); int rb_encdb_alias(const char *alias, const char *orig);
void rb_encdb_set_unicode(int index);
#pragma GCC visibility pop #pragma GCC visibility pop
#endif #endif
@ -760,14 +759,6 @@ rb_encdb_alias(const char *alias, const char *orig)
return r; return r;
} }
void
rb_encdb_set_unicode(int index)
{
rb_raw_encoding *enc = (rb_raw_encoding *)rb_enc_from_index(index);
ASSUME(enc);
enc->flags |= ONIGENC_FLAG_UNICODE;
}
static void static void
rb_enc_init(struct enc_table *enc_table) rb_enc_init(struct enc_table *enc_table)
{ {

1
internal/encoding.h
View File

@ -24,7 +24,6 @@ int rb_encdb_dummy(const char *name);
void rb_encdb_declare(const char *name); void rb_encdb_declare(const char *name);
void rb_enc_set_base(const char *name, const char *orig); void rb_enc_set_base(const char *name, const char *orig);
int rb_enc_set_dummy(int index); int rb_enc_set_dummy(int index);
void rb_encdb_set_unicode(int index);
PUREFUNC(int rb_data_is_encoding(VALUE obj)); PUREFUNC(int rb_data_is_encoding(VALUE obj));
#endif /* INTERNAL_ENCODING_H */ #endif /* INTERNAL_ENCODING_H */

61
string.c
View File

@ -355,40 +355,10 @@ rb_debug_rstring_null_ptr(const char *func)
/* symbols for [up|down|swap]case/capitalize options */ /* symbols for [up|down|swap]case/capitalize options */
static VALUE sym_ascii, sym_turkic, sym_lithuanian, sym_fold; static VALUE sym_ascii, sym_turkic, sym_lithuanian, sym_fold;
static rb_encoding *
get_actual_encoding(const int encidx, VALUE str)
{
const unsigned char *q;
switch (encidx) {
case ENCINDEX_UTF_16:
if (RSTRING_LEN(str) < 2) break;
q = (const unsigned char *)RSTRING_PTR(str);
if (q[0] == 0xFE && q[1] == 0xFF) {
return rb_enc_get_from_index(ENCINDEX_UTF_16BE);
}
if (q[0] == 0xFF && q[1] == 0xFE) {
return rb_enc_get_from_index(ENCINDEX_UTF_16LE);
}
return rb_ascii8bit_encoding();
case ENCINDEX_UTF_32:
if (RSTRING_LEN(str) < 4) break;
q = (const unsigned char *)RSTRING_PTR(str);
if (q[0] == 0 && q[1] == 0 && q[2] == 0xFE && q[3] == 0xFF) {
return rb_enc_get_from_index(ENCINDEX_UTF_32BE);
}
if (q[3] == 0 && q[2] == 0 && q[1] == 0xFE && q[0] == 0xFF) {
return rb_enc_get_from_index(ENCINDEX_UTF_32LE);
}
return rb_ascii8bit_encoding();
}
return rb_enc_from_index(encidx);
}
static rb_encoding * static rb_encoding *
get_encoding(VALUE str) get_encoding(VALUE str)
{ {
return get_actual_encoding(ENCODING_GET(str), str); return rb_enc_from_index(ENCODING_GET(str));
} }
static void static void
@ -832,21 +802,15 @@ rb_enc_cr_str_exact_copy(VALUE dest, VALUE src)
} }
static int static int
enc_coderange_scan(VALUE str, rb_encoding *enc, int encidx) enc_coderange_scan(VALUE str, rb_encoding *enc)
{ {
if (rb_enc_mbminlen(enc) > 1 && rb_enc_dummy_p(enc) && return coderange_scan(RSTRING_PTR(str), RSTRING_LEN(str), enc);
rb_enc_mbminlen(enc = get_actual_encoding(encidx, str)) == 1) {
return ENC_CODERANGE_BROKEN;
}
else {
return coderange_scan(RSTRING_PTR(str), RSTRING_LEN(str), enc);
}
} }
int int
rb_enc_str_coderange_scan(VALUE str, rb_encoding *enc) rb_enc_str_coderange_scan(VALUE str, rb_encoding *enc)
{ {
return enc_coderange_scan(str, enc, rb_enc_to_index(enc)); return enc_coderange_scan(str, enc);
} }
int int
@ -855,9 +819,7 @@ rb_enc_str_coderange(VALUE str)
int cr = ENC_CODERANGE(str); int cr = ENC_CODERANGE(str);
if (cr == ENC_CODERANGE_UNKNOWN) { if (cr == ENC_CODERANGE_UNKNOWN) {
int encidx = ENCODING_GET(str); cr = enc_coderange_scan(str, get_encoding(str));
rb_encoding *enc = rb_enc_from_index(encidx);
cr = enc_coderange_scan(str, enc, encidx);
ENC_CODERANGE_SET(str, cr); ENC_CODERANGE_SET(str, cr);
} }
return cr; return cr;
@ -1123,7 +1085,7 @@ is_enc_ascii_string(VALUE str, rb_encoding *enc)
int encidx = rb_enc_to_index(enc); int encidx = rb_enc_to_index(enc);
if (rb_enc_get_index(str) == encidx) if (rb_enc_get_index(str) == encidx)
return is_ascii_string(str); return is_ascii_string(str);
return enc_coderange_scan(str, enc, encidx) == ENC_CODERANGE_7BIT; return enc_coderange_scan(str, enc) == ENC_CODERANGE_7BIT;
} }
VALUE VALUE
@ -6730,7 +6692,7 @@ VALUE
rb_str_inspect(VALUE str) rb_str_inspect(VALUE str)
{ {
int encidx = ENCODING_GET(str); int encidx = ENCODING_GET(str);
rb_encoding *enc = rb_enc_from_index(encidx), *actenc; rb_encoding *enc = rb_enc_from_index(encidx);
const char *p, *pend, *prev; const char *p, *pend, *prev;
char buf[CHAR_ESC_LEN + 1]; char buf[CHAR_ESC_LEN + 1];
VALUE result = rb_str_buf_new(0); VALUE result = rb_str_buf_new(0);
@ -6745,11 +6707,6 @@ rb_str_inspect(VALUE str)
p = RSTRING_PTR(str); pend = RSTRING_END(str); p = RSTRING_PTR(str); pend = RSTRING_END(str);
prev = p; prev = p;
actenc = get_actual_encoding(encidx, str);
if (actenc != enc) {
enc = actenc;
if (unicode_p) unicode_p = rb_enc_unicode_p(enc);
}
while (p < pend) { while (p < pend) {
unsigned int c, cc; unsigned int c, cc;
int n; int n;
@ -9374,7 +9331,7 @@ rb_str_each_grapheme_cluster_size(VALUE str, VALUE args, VALUE eobj)
{ {
size_t grapheme_cluster_count = 0; size_t grapheme_cluster_count = 0;
regex_t *reg_grapheme_cluster = NULL; regex_t *reg_grapheme_cluster = NULL;
rb_encoding *enc = rb_enc_from_index(ENCODING_GET(str)); rb_encoding *enc = get_encoding(str);
const char *ptr, *end; const char *ptr, *end;
if (!rb_enc_unicode_p(enc)) { if (!rb_enc_unicode_p(enc)) {
@ -9402,7 +9359,7 @@ rb_str_enumerate_grapheme_clusters(VALUE str, VALUE ary)
{ {
VALUE orig = str; VALUE orig = str;
regex_t *reg_grapheme_cluster = NULL; regex_t *reg_grapheme_cluster = NULL;
rb_encoding *enc = rb_enc_from_index(ENCODING_GET(str)); rb_encoding *enc = get_encoding(str);
const char *ptr0, *ptr, *end; const char *ptr0, *ptr, *end;
if (!rb_enc_unicode_p(enc)) { if (!rb_enc_unicode_p(enc)) {

40
test/ruby/test_m17n.rb
View File

@ -226,38 +226,16 @@ class TestM17N < Test::Unit::TestCase
end end
end end
STR_WITHOUT_BOM = "\u3042".freeze def test_utf_dummy_are_like_regular_dummy_encodings
STR_WITH_BOM = "\uFEFF\u3042".freeze [Encoding::UTF_16, Encoding::UTF_32].each do |enc|
bug8940 = '[ruby-core:59757] [Bug #8940]' s = "\u3042".encode("UTF-32BE")
bug9415 = '[ruby-dev:47895] [Bug #9415]' assert_equal(s.dup.force_encoding("ISO-2022-JP").inspect, s.dup.force_encoding(enc).inspect)
%w/UTF-16 UTF-32/.each do |enc| s = "\x00\x00\xFE\xFF"
%w/BE LE/.each do |endian| assert_equal(s.dup.force_encoding("ISO-2022-JP").inspect, s.dup.force_encoding(enc).inspect)
bom = "\uFEFF".encode("#{enc}#{endian}").force_encoding(enc)
define_method("test_utf_16_32_inspect(#{enc}#{endian})") do assert_equal [0, 0, 254, 255], "\x00\x00\xFE\xFF".force_encoding(enc).codepoints
s = STR_WITHOUT_BOM.encode(enc + endian) assert_equal 0, "\x00\x00\xFE\xFF".force_encoding(enc).ord
# When a UTF-16/32 string doesn't have a BOM, assert_equal 255, "\xFF\xFE\x00\x00".force_encoding(enc).ord
# inspect as a dummy encoding string.
assert_equal(s.dup.force_encoding("ISO-2022-JP").inspect,
s.dup.force_encoding(enc).inspect)
assert_normal_exit("#{bom.b.dump}.force_encoding('#{enc}').inspect", bug8940)
end
define_method("test_utf_16_32_codepoints(#{enc}#{endian})") do
assert_equal([0xFEFF], bom.codepoints, bug9415)
end
define_method("test_utf_16_32_ord(#{enc}#{endian})") do
assert_equal(0xFEFF, bom.ord, bug9415)
end
define_method("test_utf_16_32_inspect(#{enc}#{endian}-BOM)") do
s = STR_WITH_BOM.encode(enc + endian)
# When a UTF-16/32 string has a BOM,
# inspect as a particular encoding string.
assert_equal(s.inspect,
s.dup.force_encoding(enc).inspect)
end
end end
end end

12
test/ruby/test_transcode.rb
View File

@ -2232,12 +2232,12 @@ class TestTranscode < Test::Unit::TestCase
assert_equal("U+3042", "\u{3042}".encode("US-ASCII", fallback: fallback)) assert_equal("U+3042", "\u{3042}".encode("US-ASCII", fallback: fallback))
end end
bug8940 = '[ruby-core:57318] [Bug #8940]' def test_pseudo_encoding_inspect
%w[UTF-32 UTF-16].each do |enc| s = 'aaa'.encode "UTF-16"
define_method("test_pseudo_encoding_inspect(#{enc})") do assert_equal '"\xFE\xFF\x00\x61\x00\x61\x00\x61"', s.inspect
assert_normal_exit("'aaa'.encode('#{enc}').inspect", bug8940)
assert_equal(4, 'aaa'.encode(enc).length, "should count in #{enc} with BOM") s = 'aaa'.encode "UTF-32"
end assert_equal '"\x00\x00\xFE\xFF\x00\x00\x00\x61\x00\x00\x00\x61\x00\x00\x00\x61"', s.inspect
end end
def test_encode_with_invalid_chars def test_encode_with_invalid_chars