1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ruby/irb] Fix process_continue(rename to should_continue?) and

Browse Source 
check_code_block(rename to check_code_syntax)
(https://github.com/ruby/irb/pull/611)

https://github.com/ruby/irb/commit/b7f4bfaaa4
This commit is contained in:
tomoya ishida 2023-06-25 14:12:12 +09:00 committed by git
parent 406799cae8
commit 00216c8aa0
3 changed files with 94 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/irb/cmd/show_source.rb
View File

@ -58,9 +58,9 @@ module IRB
tokens.chunk { |tok| tok.pos[0] }.each do |lnum, chunk|
code = lines[0..lnum].join
prev_tokens.concat chunk
continue = lex.process_continue(prev_tokens)
code_block_open = lex.check_code_block(code, prev_tokens)
if !continue && !code_block_open
continue = lex.should_continue?(prev_tokens)
syntax = lex.check_code_syntax(code)
if !continue && syntax == :valid
return first_line + lnum
end
end

89
lib/irb/ruby-lex.rb
View File

@ -85,7 +85,7 @@ class RubyLex
# Avoid appending duplicated token. Tokens that include "\n" like multiline tstring_content can exist in multiple lines.
tokens_until_line << token if token != tokens_until_line.last
end
continue = process_continue(tokens_until_line)
continue = should_continue?(tokens_until_line)
prompt(next_opens, continue, line_num_offset)
end
end
@ -196,7 +196,16 @@ class RubyLex
end
def code_terminated?(code, tokens, opens)
opens.empty? && !process_continue(tokens) && !check_code_block(code, tokens)
case check_code_syntax(code)
when :unrecoverable_error
true
when :recoverable_error
false
when :other_error
opens.empty? && !should_continue?(tokens)
when :valid
!should_continue?(tokens)
end
end
def save_prompt_to_context_io(opens, continue, line_num_offset)
@ -227,7 +236,7 @@ class RubyLex
return code if terminated
line_offset += 1
continue = process_continue(tokens)
continue = should_continue?(tokens)
save_prompt_to_context_io(opens, continue, line_offset)
end
end
@ -246,29 +255,33 @@ class RubyLex
end
end
def process_continue(tokens)
# last token is always newline
if tokens.size >= 2 and tokens[-2].event == :on_regexp_end
# end of regexp literal
return false
elsif tokens.size >= 2 and tokens[-2].event == :on_semicolon
return false
elsif tokens.size >= 2 and tokens[-2].event == :on_kw and ['begin', 'else', 'ensure'].include?(tokens[-2].tok)
return false
elsif !tokens.empty? and tokens.last.tok == "\\\n"
return true
elsif tokens.size >= 1 and tokens[-1].event == :on_heredoc_end # "EOH\n"
return false
elsif tokens.size >= 2 and tokens[-2].state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) and tokens[-2].tok !~ /\A\.\.\.?\z/
# end of literal except for regexp
# endless range at end of line is not a continue
return true
def should_continue?(tokens)
# Look at the last token and check if IRB need to continue reading next line.
# Example code that should continue: `a\` `a +` `a.`
# Trailing spaces, newline, comments are skipped
return true if tokens.last&.event == :on_sp && tokens.last.tok == "\\\n"
tokens.reverse_each do |token|
case token.event
when :on_sp, :on_nl, :on_ignored_nl, :on_comment, :on_embdoc_beg, :on_embdoc, :on_embdoc_end
# Skip
when :on_regexp_end, :on_heredoc_end, :on_semicolon
# State is EXPR_BEG but should not continue
return false
else
# Endless range should not continue
return false if token.event == :on_op && token.tok.match?(/\A\.\.\.?\z/)
# EXPR_DOT and most of the EXPR_BEG should continue
return token.state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_DOT)
end
end
false
end
def check_code_block(code, tokens)
return true if tokens.empty?
def check_code_syntax(code)
lvars_code = RubyLex.generate_local_variables_assign_code(@context.local_variables)
code = "#{lvars_code}\n#{code}"
begin # check if parser error are available
verbose, $VERBOSE = $VERBOSE, nil
@ -287,6 +300,7 @@ class RubyLex
end
rescue EncodingError
# This is for a hash with invalid encoding symbol, {"\xAE": 1}
:unrecoverable_error
rescue SyntaxError => e
case e.message
when /unterminated (?:string|regexp) meets end of file/
@ -299,7 +313,7 @@ class RubyLex
#
# example:
# '
return true
return :recoverable_error
when /syntax error, unexpected end-of-input/
# "syntax error, unexpected end-of-input, expecting keyword_end"
#
@ -309,7 +323,7 @@ class RubyLex
# if false
# fuga
# end
return true
return :recoverable_error
when /syntax error, unexpected keyword_end/
# "syntax error, unexpected keyword_end"
#
@ -319,41 +333,26 @@ class RubyLex
#
# example:
# end
return false
return :unrecoverable_error
when /syntax error, unexpected '\.'/
# "syntax error, unexpected '.'"
#
# example:
# .
return false
return :unrecoverable_error
when /unexpected tREGEXP_BEG/
# "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
#
# example:
# method / f /
return false
return :unrecoverable_error
else
return :other_error
end
ensure
$VERBOSE = verbose
end
last_lex_state = tokens.last.state
if last_lex_state.allbits?(Ripper::EXPR_BEG)
return false
elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
return true
elsif last_lex_state.allbits?(Ripper::EXPR_CLASS)
return true
elsif last_lex_state.allbits?(Ripper::EXPR_FNAME)
return true
elsif last_lex_state.allbits?(Ripper::EXPR_VALUE)
return true
elsif last_lex_state.allbits?(Ripper::EXPR_ARG)
return false
end
false
:valid
end
def calc_indent_level(opens)

56
test/irb/test_ruby_lex.rb
View File

@ -82,25 +82,33 @@ module TestIRB
end
def assert_indent_level(lines, expected, local_variables: [])
indent_level, _code_block_open = check_state(lines, local_variables: local_variables)
indent_level, _continue, _code_block_open = check_state(lines, local_variables: local_variables)
error_message = "Calculated the wrong number of indent level for:\n #{lines.join("\n")}"
assert_equal(expected, indent_level, error_message)
end
def assert_should_continue(lines, expected, local_variables: [])
_indent_level, continue, _code_block_open = check_state(lines, local_variables: local_variables)
error_message = "Wrong result of should_continue for:\n #{lines.join("\n")}"
assert_equal(expected, continue, error_message)
end
def assert_code_block_open(lines, expected, local_variables: [])
_indent_level, code_block_open = check_state(lines, local_variables: local_variables)
_indent_level, _continue, code_block_open = check_state(lines, local_variables: local_variables)
error_message = "Wrong result of code_block_open for:\n #{lines.join("\n")}"
assert_equal(expected, code_block_open, error_message)
end
def check_state(lines, local_variables: [])
context = build_context(local_variables)
tokens = RubyLex.ripper_lex_without_warning(lines.join("\n"), context: context)
code = lines.join("\n")
tokens = RubyLex.ripper_lex_without_warning(code, context: context)
opens = IRB::NestingParser.open_tokens(tokens)
ruby_lex = RubyLex.new(context)
indent_level = ruby_lex.calc_indent_level(opens)
code_block_open = !opens.empty? || ruby_lex.process_continue(tokens)
[indent_level, code_block_open]
continue = ruby_lex.should_continue?(tokens)
terminated = ruby_lex.code_terminated?(code, tokens, opens)
[indent_level, continue, !terminated]
end
def test_interpolate_token_with_heredoc_and_unclosed_embexpr
@ -235,7 +243,7 @@ module TestIRB
def test_endless_range_at_end_of_line
input_with_prompt = [
PromptRow.new('001:0: :> ', %q(a = 3..)),
PromptRow.new('002:0: :* ', %q()),
PromptRow.new('002:0: :> ', %q()),
]
lines = input_with_prompt.map(&:content)
@ -256,7 +264,7 @@ module TestIRB
PromptRow.new('009:0:]:* ', %q(B)),
PromptRow.new('010:0:]:* ', %q(})),
PromptRow.new('011:0: :> ', %q(])),
PromptRow.new('012:0: :* ', %q()),
PromptRow.new('012:0: :> ', %q()),
]
lines = input_with_prompt.map(&:content)
@ -285,9 +293,9 @@ module TestIRB
def test_backtick_method
input_with_prompt = [
PromptRow.new('001:0: :> ', %q(self.`(arg))),
PromptRow.new('002:0: :* ', %q()),
PromptRow.new('002:0: :> ', %q()),
PromptRow.new('003:0: :> ', %q(def `(); end)),
PromptRow.new('004:0: :* ', %q()),
PromptRow.new('004:0: :> ', %q()),
]
lines = input_with_prompt.map(&:content)
@ -777,6 +785,36 @@ module TestIRB
assert_dynamic_prompt(lines, expected_prompt_list)
end
def test_should_continue
assert_should_continue(['a'], false)
assert_should_continue(['/a/'], false)
assert_should_continue(['a;'], false)
assert_should_continue(['<<A', 'A'], false)
assert_should_continue(['a...'], false)
assert_should_continue(['a\\', ''], true)
assert_should_continue(['a.'], true)
assert_should_continue(['a+'], true)
assert_should_continue(['a; #comment', '', '=begin', 'embdoc', '=end', ''], false)
assert_should_continue(['a+ #comment', '', '=begin', 'embdoc', '=end', ''], true)
end
def test_code_block_open_with_should_continue
# syntax ok
assert_code_block_open(['a'], false) # continue: false
assert_code_block_open(['a\\', ''], true) # continue: true
# recoverable syntax error code is not terminated
assert_code_block_open(['a+', ''], true)
# unrecoverable syntax error code is terminated
assert_code_block_open(['.; a+', ''], false)
# other syntax error that failed to determine if it is recoverable or not
assert_code_block_open(['@; a'], false)
assert_code_block_open(['@; a+'], true)
assert_code_block_open(['@; (a'], true)
end
def test_broken_percent_literal
tokens = RubyLex.ripper_lex_without_warning('%wwww')
pos_to_index = {}