1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Bug #21202] Fix wrong token concat while tokenizing nested unterminated heredoc (#13000)

Browse Source
This commit is contained in:
tomoya ishida 2025-03-29 20:46:43 +09:00 committed by GitHub
parent 549c7fe29d
commit a4a6019550
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
Notes: git 2025-03-29 11:47:11 +00:00 
Merged-By: tompng <tomoyapenguin@gmail.com>
2 changed files with 46 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
parse.y
View File

@ -9036,7 +9036,7 @@ here_document(struct parser_params *p, rb_strterm_heredoc_t *here)
if (!has_delayed_token(p)) {
dispatch_scan_event(p, tSTRING_CONTENT);
}
else {
else if (p->delayed.end_line + 1 == p->ruby_sourceline) {
if ((len = p->lex.pcur - p->lex.ptok) > 0) {
if (!(func & STR_FUNC_REGEXP)) {
int cr = ENC_CODERANGE_UNKNOWN;
@ -9051,6 +9051,10 @@ here_document(struct parser_params *p, rb_strterm_heredoc_t *here)
}
dispatch_delayed_token(p, tSTRING_CONTENT);
}
else {
dispatch_delayed_token(p, tSTRING_CONTENT);
dispatch_scan_event(p, tSTRING_CONTENT);
}
lex_goto_eol(p);
#endif
heredoc_restore(p, &p->lex.strterm->u.heredoc);

41
test/ripper/test_lexer.rb
View File

@ -344,6 +344,47 @@ world"
]
assert_lexer(expected, code)
code = <<~'HEREDOC'
<<H1
#{<<H2}a
H2
b
HEREDOC
expected = [
[[1, 0], :on_heredoc_beg, "<<H1", state(:EXPR_BEG)],
[[1, 4], :on_nl, "\n", state(:EXPR_BEG)],
[[2, 0], :on_embexpr_beg, "\#{", state(:EXPR_BEG)],
[[2, 2], :on_heredoc_beg, "<<H2", state(:EXPR_BEG)],
[[2, 6], :on_embexpr_end, "}", state(:EXPR_END)],
[[2, 7], :on_tstring_content, "a\n", state(:EXPR_BEG)],
[[3, 0], :on_heredoc_end, "H2\n", state(:EXPR_BEG)],
[[4, 0], :on_tstring_content, "b\n", state(:EXPR_BEG)]
]
assert_lexer(expected, code)
code = <<~'HEREDOC'
<<H1
#{<<H2}a
H2
b
c
HEREDOC
expected = [
[[1, 0], :on_heredoc_beg, "<<H1", state(:EXPR_BEG)],
[[1, 4], :on_nl, "\n", state(:EXPR_BEG)],
[[2, 0], :on_embexpr_beg, "\#{", state(:EXPR_BEG)],
[[2, 2], :on_heredoc_beg, "<<H2", state(:EXPR_BEG)],
[[2, 6], :on_embexpr_end, "}", state(:EXPR_END)],
[[2, 7], :on_tstring_content, "a\n", state(:EXPR_BEG)],
[[3, 0], :on_heredoc_end, "H2\n", state(:EXPR_BEG)],
[[4, 0], :on_tstring_content, "b\nc\n", state(:EXPR_BEG)]
]
assert_lexer(expected, code)
end
def test_invalid_escape_ctrl_mbchar