1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Bug #19563] Yield words separators per lines

Browse Source 
So that newlines across a here-doc terminator will be separated
tokens.

Cf. https://github.com/ruby/irb/pull/558
This commit is contained in:
Nobuyoshi Nakada 2023-04-07 23:13:56 +09:00 committed by GitHub
parent 4df7c3946a
commit ac8a16237c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
Notes: git 2023-04-07 14:14:21 +00:00 
Merged: https://github.com/ruby/ruby/pull/7675

Merged-By: nobu <nobu@ruby-lang.org>
2 changed files with 16 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
parse.y
View File

@ -5025,7 +5025,11 @@ regexp : tREGEXP_BEG regexp_contents tREGEXP_END
} }
; ;
words : tWORDS_BEG ' ' word_list tSTRING_END words_sep : ' ' {}
| words_sep ' '
;
words : tWORDS_BEG words_sep word_list tSTRING_END
{ {
/*%%%*/ /*%%%*/
$$ = make_list($3, &@$); $$ = make_list($3, &@$);
@ -5041,7 +5045,7 @@ word_list : /* none */
/*% %*/ /*% %*/
/*% ripper: words_new! %*/ /*% ripper: words_new! %*/
} }
| word_list word ' ' | word_list word words_sep
{ {
/*%%%*/ /*%%%*/
$$ = list_append(p, $1, evstr2dstr(p, $2)); $$ = list_append(p, $1, evstr2dstr(p, $2));
@ -5061,7 +5065,7 @@ word : string_content
} }
; ;
symbols : tSYMBOLS_BEG ' ' symbol_list tSTRING_END symbols : tSYMBOLS_BEG words_sep symbol_list tSTRING_END
{ {
/*%%%*/ /*%%%*/
$$ = make_list($3, &@$); $$ = make_list($3, &@$);
@ -5077,7 +5081,7 @@ symbol_list : /* none */
/*% %*/ /*% %*/
/*% ripper: symbols_new! %*/ /*% ripper: symbols_new! %*/
} }
| symbol_list word ' ' | symbol_list word words_sep
{ {
/*%%%*/ /*%%%*/
$$ = symbol_append(p, $1, evstr2dstr(p, $2)); $$ = symbol_append(p, $1, evstr2dstr(p, $2));
@ -5086,7 +5090,7 @@ symbol_list : /* none */
} }
; ;
qwords : tQWORDS_BEG ' ' qword_list tSTRING_END qwords : tQWORDS_BEG words_sep qword_list tSTRING_END
{ {
/*%%%*/ /*%%%*/
$$ = make_list($3, &@$); $$ = make_list($3, &@$);
@ -5095,7 +5099,7 @@ qwords : tQWORDS_BEG ' ' qword_list tSTRING_END
} }
; ;
qsymbols : tQSYMBOLS_BEG ' ' qsym_list tSTRING_END qsymbols : tQSYMBOLS_BEG words_sep qsym_list tSTRING_END
{ {
/*%%%*/ /*%%%*/
$$ = make_list($3, &@$); $$ = make_list($3, &@$);
@ -5111,7 +5115,7 @@ qword_list : /* none */
/*% %*/ /*% %*/
/*% ripper: qwords_new! %*/ /*% ripper: qwords_new! %*/
} }
| qword_list tSTRING_CONTENT ' ' | qword_list tSTRING_CONTENT words_sep
{ {
/*%%%*/ /*%%%*/
$$ = list_append(p, $1, $2); $$ = list_append(p, $1, $2);
@ -5127,7 +5131,7 @@ qsym_list : /* none */
/*% %*/ /*% %*/
/*% ripper: qsymbols_new! %*/ /*% ripper: qsymbols_new! %*/
} }
| qsym_list tSTRING_CONTENT ' ' | qsym_list tSTRING_CONTENT words_sep
{ {
/*%%%*/ /*%%%*/
$$ = symbol_append(p, $1, $2); $$ = symbol_append(p, $1, $2);
@ -7909,7 +7913,8 @@ parse_string(struct parser_params *p, rb_strterm_literal_t *quote)
} }
c = nextc(p); c = nextc(p);
if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) { if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
do {c = nextc(p);} while (ISSPACE(c)); ruby_debug_breakpoint();
while (c != '\n' && ISSPACE(c = nextc(p)));
space = 1; space = 1;
} }
if (func & STR_FUNC_LIST) { if (func & STR_FUNC_LIST) {
@ -7926,7 +7931,7 @@ parse_string(struct parser_params *p, rb_strterm_literal_t *quote)
return parser_string_term(p, func); return parser_string_term(p, func);
} }
if (space) { if (space) {
pushback(p, c); if (!ISSPACE(c)) pushback(p, c);
add_delayed_token(p, p->lex.ptok, p->lex.pcur, __LINE__); add_delayed_token(p, p->lex.ptok, p->lex.pcur, __LINE__);
return ' '; return ' ';
} }

2
test/ripper/test_scanner_events.rb
View File

@ -712,7 +712,7 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
scan('words_sep', '%w( w w w )') scan('words_sep', '%w( w w w )')
assert_equal [' ', "\n", ' ', ' '], assert_equal [' ', "\n", ' ', ' '],
scan('words_sep', "%w( w\nw w )") scan('words_sep', "%w( w\nw w )")
assert_equal ["\n\n", "\n ", ' ', ' '], assert_equal ["\n", "\n", "\n", ' ', ' ', ' '],
scan('words_sep', "%w(\n\nw\n w w )") scan('words_sep', "%w(\n\nw\n w w )")
end end