1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Bug #20924] Fix reading with delimiter in wide character encodings

Browse Source
This commit is contained in:
Nobuyoshi Nakada 2024-12-10 23:57:57 +09:00
parent 3422bfcab6
commit e90b447655
No known key found for this signature in database
GPG Key ID: 3582D74E1FEE4465
Notes: git 2024-12-10 16:14:28 +00:00 
Merged: https://github.com/ruby/ruby/pull/12304
2 changed files with 75 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
io.c
View File

@ -3849,8 +3849,33 @@ rscheck(const char *rsptr, long rslen, VALUE rs)
rb_raise(rb_eRuntimeError, "rs modified");
}
static const char *
search_delim(const char *p, long len, int delim, rb_encoding *enc)
{
if (rb_enc_mbminlen(enc) == 1) {
p = memchr(p, delim, len);
if (p) return p + 1;
}
else {
const char *end = p + len;
while (p < end) {
int r = rb_enc_precise_mbclen(p, end, enc);
if (!MBCLEN_CHARFOUND_P(r)) {
p += rb_enc_mbminlen(enc);
continue;
}
int n = MBCLEN_CHARFOUND_LEN(r);
if (rb_enc_mbc_to_codepoint(p, end, enc) == (unsigned int)delim) {
return p + n;
}
p += n;
}
}
return NULL;
}
static int
appendline(rb_io_t *fptr, int delim, VALUE *strp, long *lp)
appendline(rb_io_t *fptr, int delim, VALUE *strp, long *lp, rb_encoding *enc)
{
VALUE str = *strp;
long limit = *lp;
@ -3865,9 +3890,9 @@ appendline(rb_io_t *fptr, int delim, VALUE *strp, long *lp)
p = READ_CHAR_PENDING_PTR(fptr);
if (0 < limit && limit < searchlen)
searchlen = (int)limit;
e = memchr(p, delim, searchlen);
e = search_delim(p, searchlen, delim, enc);
if (e) {
int len = (int)(e-p+1);
int len = (int)(e-p);
if (NIL_P(str))
*strp = str = rb_str_new(p, len);
else
@ -3907,8 +3932,8 @@ appendline(rb_io_t *fptr, int delim, VALUE *strp, long *lp)
long last;
if (limit > 0 && pending > limit) pending = limit;
e = memchr(p, delim, pending);
if (e) pending = e - p + 1;
e = search_delim(p, pending, delim, enc);
if (e) pending = e - p;
if (!NIL_P(str)) {
last = RSTRING_LEN(str);
rb_str_resize(str, last + pending);
@ -4168,16 +4193,26 @@ rb_io_getline_0(VALUE rs, long limit, int chomp, rb_io_t *fptr)
rsptr = RSTRING_PTR(rs);
rslen = RSTRING_LEN(rs);
}
newline = '\n';
}
else if (rb_enc_mbminlen(enc) == 1) {
rsptr = RSTRING_PTR(rs);
newline = (unsigned char)rsptr[rslen - 1];
}
else {
rs = rb_str_encode(rs, rb_enc_from_encoding(enc), 0, Qnil);
rsptr = RSTRING_PTR(rs);
const char *e = rsptr + rslen;
const char *last = rb_enc_prev_char(rsptr, e, e, enc);
int n;
newline = rb_enc_codepoint_len(last, e, &n, enc);
if (last + n != e) rb_raise(rb_eArgError, "broken separator");
}
newline = (unsigned char)rsptr[rslen - 1];
chomp_cr = chomp && rslen == 1 && newline == '\n';
chomp_cr = chomp && newline == '\n' && rslen == rb_enc_mbminlen(enc);
}
/* MS - Optimization */
while ((c = appendline(fptr, newline, &str, &limit)) != EOF) {
while ((c = appendline(fptr, newline, &str, &limit, enc)) != EOF) {
const char *s, *p, *pp, *e;
if (c == newline) {
@ -4198,8 +4233,8 @@ rb_io_getline_0(VALUE rs, long limit, int chomp, rb_io_t *fptr)
if (limit == 0) {
s = RSTRING_PTR(str);
p = RSTRING_END(str);
pp = rb_enc_left_char_head(s, p-1, p, enc);
if (extra_limit &&
pp = rb_enc_prev_char(s, p, p, enc);
if (extra_limit && pp &&
MBCLEN_NEEDMORE_P(rb_enc_precise_mbclen(pp, p, enc))) {
/* relax the limit while incomplete character.
* extra_limit limits the relax length */

30
test/ruby/test_io.rb
View File

@ -2036,6 +2036,36 @@ class TestIO < Test::Unit::TestCase
}
end
def test_readline_limit_nonascii
mkcdtmpdir do
i = 0
File.open("text#{i+=1}", "w+:utf-8") do |f|
f.write("Test\nok\u{bf}ok\n")
f.rewind
assert_equal("Test\nok\u{bf}", f.readline("\u{bf}"))
assert_equal("ok\n", f.readline("\u{bf}"))
end
File.open("text#{i+=1}", "w+b:utf-32le") do |f|
f.write("0123456789")
f.rewind
assert_equal(4, f.readline(4).bytesize)
assert_equal(4, f.readline(3).bytesize)
end
File.open("text#{i+=1}", "w+:utf-8:utf-32le") do |f|
f.write("0123456789")
f.rewind
assert_equal(4, f.readline(4).bytesize)
assert_equal(4, f.readline(3).bytesize)
end
end
end
def test_set_lineno_readline
pipe(proc do |w|
w.puts "foo"