diff --git a/parse.y b/parse.y index 301266c36f..ec9a82e534 100644 --- a/parse.y +++ b/parse.y @@ -9813,7 +9813,7 @@ parser_yylex(struct parser_params *p) #endif /* Set location for end-of-input because dispatch_scan_event is not called. */ RUBY_SET_YYLLOC(*p->yylloc); - return 0; + return END_OF_INPUT; /* white spaces */ case '\r': @@ -9986,7 +9986,7 @@ parser_yylex(struct parser_params *p) c = nextc(p); if (c == -1) { compile_error(p, "embedded document meets end of file"); - return 0; + return END_OF_INPUT; } if (c == '=' && word_match_p(p, "end", 3)) { break; @@ -10466,13 +10466,11 @@ parser_yylex(struct parser_params *p) if (was_bol(p) && whole_match_p(p, "__END__", 7, 0)) { p->ruby__end__seen = 1; p->eofp = 1; -#ifndef RIPPER - return -1; -#else +#ifdef RIPPER lex_goto_eol(p); dispatch_scan_event(p, k__END__); - return 0; #endif + return END_OF_INPUT; } newtok(p); break; @@ -10504,7 +10502,7 @@ yylex(YYSTYPE *lval, YYLTYPE *yylloc, struct parser_params *p) if (has_delayed_token(p)) dispatch_delayed_token(p, t); - else if (t != 0) + else if (t != END_OF_INPUT) dispatch_scan_event(p, t); return t; diff --git a/test/ruby/test_ast.rb b/test/ruby/test_ast.rb index e43dac28b3..875cf7138e 100644 --- a/test/ruby/test_ast.rb +++ b/test/ruby/test_ast.rb @@ -624,6 +624,27 @@ dummy assert_equal("def test_keep_script_lines_for_of\n", node_method.source.lines.first) end + def test_keep_tokens_for_parse + node = RubyVM::AbstractSyntaxTree.parse(<<~END, keep_tokens: true) + 1.times do + end + __END__ + dummy + END + + expected = [ + [:tINTEGER, "1"], + [:".", "."], + [:tIDENTIFIER, "times"], + [:tSP, " "], + [:keyword_do, "do"], + [:tIGNORED_NL, "\n"], + [:keyword_end, "end"], + [:nl, "\n"], + ] + assert_equal(expected, node.all_tokens.map { [_2, _3]}) + end + def test_encoding_with_keep_script_lines # Stop a warning "possibly useless use of a literal in void context" verbose_bak, $VERBOSE = $VERBOSE, nil