[ruby/yarp] Fix first method param lex failures

When Ripper encounters a method parameter that is the first
parameter and is an identifier and it shadows a local scope, it
incorrectly marks it as END|LABEL (because it think it's a local).

We need to account for that in the lex compat in order to properly
compare.

https://github.com/ruby/yarp/commit/15f725a1b1
This commit is contained in:
Kevin Newton 2023-08-23 13:48:39 -04:00 committed by git
parent 5766fb7266
commit 24bcd49473

View File

@ -252,6 +252,23 @@ module YARP
end
end
# If we have an identifier that follows a method name like:
#
# def foo bar
#
# then Ripper will mark bar as END|LABEL if there is a local in a parent
# scope named bar because it hasn't pushed the local table yet. We do this
# more accurately, so we need to allow comparing against both END and
# END|LABEL.
class ParamToken < Token
def ==(other)
(self[0...-1] == other[0...-1]) && (
(other[3] == Ripper::EXPR_END) ||
(other[3] == Ripper::EXPR_END | Ripper::EXPR_LABEL)
)
end
end
# A heredoc in this case is a list of tokens that belong to the body of the
# heredoc that should be appended onto the list of tokens when the heredoc
# closes.
@ -586,7 +603,17 @@ module YARP
# want to bother comparing the state on them.
HeredocEndToken.new([[lineno, column], event, value, lex_state])
when :on_embexpr_end, :on_ident
if lex_state == Ripper::EXPR_END | Ripper::EXPR_LABEL
if lex_state == Ripper::EXPR_END
# If we have an identifier that follows a method name like:
#
# def foo bar
#
# then Ripper will mark bar as END|LABEL if there is a local in a
# parent scope named bar because it hasn't pushed the local table
# yet. We do this more accurately, so we need to allow comparing
# against both END and END|LABEL.
ParamToken.new([[lineno, column], event, value, lex_state])
elsif lex_state == Ripper::EXPR_END | Ripper::EXPR_LABEL
# In the event that we're comparing identifiers, we're going to
# allow a little divergence. Ripper doesn't account for local
# variables introduced through named captures in regexes, and we
@ -630,11 +657,18 @@ module YARP
end
Token.new([[lineno, column], event, value, lex_state])
when :on_eof
prev_token = result_value[index-1][0]
if prev_token.type == :COMMENT && prev_token.location.end_offset < token.location.start_offset
tokens << Token.new([[lineno, 0], :on_nl, source.byteslice(result_value[index-1].first.location.end_offset...token.location.start_offset), lex_state])
previous_token = result_value[index - 1][0]
# If we're at the end of the file and the previous token was a
# comment and there is still whitespace after the comment, then
# Ripper will append a on_nl token (even though there isn't
# necessarily a newline). We mirror that here.
start_offset = previous_token.location.end_offset
end_offset = token.location.start_offset
if previous_token.type == :COMMENT && start_offset < end_offset
tokens << Token.new([[lineno, 0], :on_nl, source.byteslice(start_offset...end_offset), lex_state])
end
Token.new([[lineno, column], event, value, lex_state])
@ -721,7 +755,8 @@ module YARP
end
end
tokens.reject! { |t| t.event == :on_eof }
# Drop the EOF token from the list
tokens = tokens[0...-1]
# We sort by location to compare against Ripper's output
tokens.sort_by!(&:location)