[ruby/irb] Store context in RubyLex

Some background for this refactor:

1. Through a RubyLex instance's lifetime, the context passed to its methods
   should be the same.
   Given that `Context` is only initialised in `Irb#initialize`,
   this should be true.

2. When `RubyLex` is initialised, the context object should be accessible.
   This is also true in all 3 of `RubyLex.new`'s invocations.

With the above observations, we should be able to store the context in `RubyLex`
as an instance variable. And doing so will make `RubyLex`'s instance methods
easier to use and maintain.

https://github.com/ruby/irb/commit/5c8d3df2df
This commit is contained in:
Stan Lo 2023-01-10 20:43:33 +00:00 committed by git
parent 2082ba7c69
commit cb9b885e78
4 changed files with 48 additions and 44 deletions

View File

@ -468,7 +468,7 @@ module IRB
@context = Context.new(self, workspace, input_method) @context = Context.new(self, workspace, input_method)
@context.main.extend ExtendCommandBundle @context.main.extend ExtendCommandBundle
@signal_status = :IN_IRB @signal_status = :IN_IRB
@scanner = RubyLex.new @scanner = RubyLex.new(@context)
end end
# A hook point for `debug` command's TracePoint after :IRB_EXIT as well as its clean-up # A hook point for `debug` command's TracePoint after :IRB_EXIT as well as its clean-up
@ -538,7 +538,7 @@ module IRB
@context.io.prompt @context.io.prompt
end end
@scanner.set_input(@context.io, context: @context) do @scanner.set_input(@context.io) do
signal_status(:IN_INPUT) do signal_status(:IN_INPUT) do
if l = @context.io.gets if l = @context.io.gets
print l if @context.verbose? print l if @context.verbose?
@ -556,9 +556,9 @@ module IRB
end end
end end
@scanner.set_auto_indent(@context) if @context.auto_indent_mode @scanner.set_auto_indent
@scanner.each_top_level_statement(@context) do |line, line_no| @scanner.each_top_level_statement do |line, line_no|
signal_status(:IN_EVAL) do signal_status(:IN_EVAL) do
begin begin
line.untaint if RUBY_VERSION < '2.7' line.untaint if RUBY_VERSION < '2.7'

View File

@ -40,15 +40,15 @@ module IRB
file, line = receiver.method(method).source_location if receiver.respond_to?(method) file, line = receiver.method(method).source_location if receiver.respond_to?(method)
end end
if file && line if file && line
Source.new(file: file, first_line: line, last_line: find_end(file, line)) Source.new(file: file, first_line: line, last_line: find_end(file, line, irb_context))
end end
end end
private private
def find_end(file, first_line) def find_end(file, first_line, irb_context)
return first_line unless File.exist?(file) return first_line unless File.exist?(file)
lex = RubyLex.new lex = RubyLex.new(irb_context)
lines = File.read(file).lines[(first_line - 1)..-1] lines = File.read(file).lines[(first_line - 1)..-1]
tokens = RubyLex.ripper_lex_without_warning(lines.join) tokens = RubyLex.ripper_lex_without_warning(lines.join)
prev_tokens = [] prev_tokens = []

View File

@ -16,7 +16,8 @@ class RubyLex
end end
end end
def initialize def initialize(context)
@context = context
@exp_line_no = @line_no = 1 @exp_line_no = @line_no = 1
@indent = 0 @indent = 0
@continue = false @continue = false
@ -42,13 +43,13 @@ class RubyLex
end end
# io functions # io functions
def set_input(io, context:, &block) def set_input(io, &block)
@io = io @io = io
if @io.respond_to?(:check_termination) if @io.respond_to?(:check_termination)
@io.check_termination do |code| @io.check_termination do |code|
if Reline::IOGate.in_pasting? if Reline::IOGate.in_pasting?
lex = RubyLex.new lex = RubyLex.new(@context)
rest = lex.check_termination_in_prev_line(code, context: context) rest = lex.check_termination_in_prev_line(code)
if rest if rest
Reline.delete_text Reline.delete_text
rest.bytes.reverse_each do |c| rest.bytes.reverse_each do |c|
@ -61,13 +62,13 @@ class RubyLex
else else
# Accept any single-line input for symbol aliases or commands that transform args # Accept any single-line input for symbol aliases or commands that transform args
command = code.split(/\s/, 2).first command = code.split(/\s/, 2).first
if context.symbol_alias?(command) || context.transform_args?(command) if @context.symbol_alias?(command) || @context.transform_args?(command)
next true next true
end end
code.gsub!(/\s*\z/, '').concat("\n") code.gsub!(/\s*\z/, '').concat("\n")
tokens = self.class.ripper_lex_without_warning(code, context: context) tokens = self.class.ripper_lex_without_warning(code, context: @context)
ltype, indent, continue, code_block_open = check_state(code, tokens, context: context) ltype, indent, continue, code_block_open = check_state(code, tokens)
if ltype or indent > 0 or continue or code_block_open if ltype or indent > 0 or continue or code_block_open
false false
else else
@ -80,7 +81,7 @@ class RubyLex
@io.dynamic_prompt do |lines| @io.dynamic_prompt do |lines|
lines << '' if lines.empty? lines << '' if lines.empty?
result = [] result = []
tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: context) tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: @context)
code = String.new code = String.new
partial_tokens = [] partial_tokens = []
unprocessed_tokens = [] unprocessed_tokens = []
@ -93,7 +94,7 @@ class RubyLex
t_str.each_line("\n") do |s| t_str.each_line("\n") do |s|
code << s code << s
next unless s.include?("\n") next unless s.include?("\n")
ltype, indent, continue, code_block_open = check_state(code, partial_tokens, context: context) ltype, indent, continue, code_block_open = check_state(code, partial_tokens)
result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset) result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
line_num_offset += 1 line_num_offset += 1
end end
@ -104,7 +105,7 @@ class RubyLex
end end
unless unprocessed_tokens.empty? unless unprocessed_tokens.empty?
ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens, context: context) ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens)
result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset) result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
end end
result result
@ -187,11 +188,11 @@ class RubyLex
prev_spaces prev_spaces
end end
def set_auto_indent(context) def set_auto_indent
if @io.respond_to?(:auto_indent) and context.auto_indent_mode if @io.respond_to?(:auto_indent) and @context.auto_indent_mode
@io.auto_indent do |lines, line_index, byte_pointer, is_newline| @io.auto_indent do |lines, line_index, byte_pointer, is_newline|
if is_newline if is_newline
@tokens = self.class.ripper_lex_without_warning(lines[0..line_index].join("\n"), context: context) @tokens = self.class.ripper_lex_without_warning(lines[0..line_index].join("\n"), context: @context)
prev_spaces = find_prev_spaces(line_index) prev_spaces = find_prev_spaces(line_index)
depth_difference = check_newline_depth_difference depth_difference = check_newline_depth_difference
depth_difference = 0 if depth_difference < 0 depth_difference = 0 if depth_difference < 0
@ -200,18 +201,18 @@ class RubyLex
code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join
last_line = lines[line_index]&.byteslice(0, byte_pointer) last_line = lines[line_index]&.byteslice(0, byte_pointer)
code += last_line if last_line code += last_line if last_line
@tokens = self.class.ripper_lex_without_warning(code, context: context) @tokens = self.class.ripper_lex_without_warning(code, context: @context)
check_corresponding_token_depth(lines, line_index) check_corresponding_token_depth(lines, line_index)
end end
end end
end end
end end
def check_state(code, tokens, context:) def check_state(code, tokens)
ltype = process_literal_type(tokens) ltype = process_literal_type(tokens)
indent = process_nesting_level(tokens) indent = process_nesting_level(tokens)
continue = process_continue(tokens) continue = process_continue(tokens)
lvars_code = self.class.generate_local_variables_assign_code(context.local_variables) lvars_code = self.class.generate_local_variables_assign_code(@context.local_variables)
code = "#{lvars_code}\n#{code}" if lvars_code code = "#{lvars_code}\n#{code}" if lvars_code
code_block_open = check_code_block(code, tokens) code_block_open = check_code_block(code, tokens)
[ltype, indent, continue, code_block_open] [ltype, indent, continue, code_block_open]
@ -232,13 +233,13 @@ class RubyLex
@code_block_open = false @code_block_open = false
end end
def each_top_level_statement(context) def each_top_level_statement
initialize_input initialize_input
catch(:TERM_INPUT) do catch(:TERM_INPUT) do
loop do loop do
begin begin
prompt prompt
unless l = lex(context) unless l = lex
throw :TERM_INPUT if @line == '' throw :TERM_INPUT if @line == ''
else else
@line_no += l.count("\n") @line_no += l.count("\n")
@ -268,15 +269,15 @@ class RubyLex
end end
end end
def lex(context) def lex
line = @input.call line = @input.call
if @io.respond_to?(:check_termination) if @io.respond_to?(:check_termination)
return line # multiline return line # multiline
end end
code = @line + (line.nil? ? '' : line) code = @line + (line.nil? ? '' : line)
code.gsub!(/\s*\z/, '').concat("\n") code.gsub!(/\s*\z/, '').concat("\n")
@tokens = self.class.ripper_lex_without_warning(code, context: context) @tokens = self.class.ripper_lex_without_warning(code, context: @context)
@ltype, @indent, @continue, @code_block_open = check_state(code, @tokens, context: context) @ltype, @indent, @continue, @code_block_open = check_state(code, @tokens)
line line
end end
@ -777,8 +778,8 @@ class RubyLex
end end
end end
def check_termination_in_prev_line(code, context:) def check_termination_in_prev_line(code)
tokens = self.class.ripper_lex_without_warning(code, context: context) tokens = self.class.ripper_lex_without_warning(code, context: @context)
past_first_newline = false past_first_newline = false
index = tokens.rindex do |t| index = tokens.rindex do |t|
# traverse first token before last line # traverse first token before last line

View File

@ -36,13 +36,13 @@ module TestIRB
context = build_context context = build_context
context.auto_indent_mode = true context.auto_indent_mode = true
ruby_lex = RubyLex.new() ruby_lex = RubyLex.new(context)
io = MockIO_AutoIndent.new([lines, last_line_index, byte_pointer, add_new_line]) do |auto_indent| io = MockIO_AutoIndent.new([lines, last_line_index, byte_pointer, add_new_line]) do |auto_indent|
error_message = "Calculated the wrong number of spaces for:\n #{lines.join("\n")}" error_message = "Calculated the wrong number of spaces for:\n #{lines.join("\n")}"
assert_equal(correct_space_count, auto_indent, error_message) assert_equal(correct_space_count, auto_indent, error_message)
end end
ruby_lex.set_input(io, context: context) ruby_lex.set_input(io)
ruby_lex.set_auto_indent(context) ruby_lex.set_auto_indent
end end
def assert_nesting_level(lines, expected, local_variables: []) def assert_nesting_level(lines, expected, local_variables: [])
@ -58,14 +58,14 @@ module TestIRB
end end
def ruby_lex_for_lines(lines, local_variables: []) def ruby_lex_for_lines(lines, local_variables: [])
ruby_lex = RubyLex.new()
context = build_context(local_variables) context = build_context(local_variables)
ruby_lex = RubyLex.new(context)
io = proc{ lines.join("\n") } io = proc{ lines.join("\n") }
ruby_lex.set_input(io, context: context) do ruby_lex.set_input(io) do
lines.join("\n") lines.join("\n")
end end
ruby_lex.lex(context) ruby_lex.lex
ruby_lex ruby_lex
end end
@ -633,7 +633,8 @@ module TestIRB
def assert_dynamic_prompt(lines, expected_prompt_list) def assert_dynamic_prompt(lines, expected_prompt_list)
pend if RUBY_ENGINE == 'truffleruby' pend if RUBY_ENGINE == 'truffleruby'
ruby_lex = RubyLex.new() context = build_context
ruby_lex = RubyLex.new(context)
io = MockIO_DynamicPrompt.new(lines) do |prompt_list| io = MockIO_DynamicPrompt.new(lines) do |prompt_list|
error_message = <<~EOM error_message = <<~EOM
Expected dynamic prompt: Expected dynamic prompt:
@ -647,8 +648,7 @@ module TestIRB
ruby_lex.set_prompt do |ltype, indent, continue, line_no| ruby_lex.set_prompt do |ltype, indent, continue, line_no|
'%03d:%01d:%1s:%s ' % [line_no, indent, ltype, continue ? '*' : '>'] '%03d:%01d:%1s:%s ' % [line_no, indent, ltype, continue ? '*' : '>']
end end
context = build_context ruby_lex.set_input(io)
ruby_lex.set_input(io, context: context)
end end
def test_dyanmic_prompt def test_dyanmic_prompt
@ -751,9 +751,10 @@ module TestIRB
end end
def test_unterminated_heredoc_string_literal def test_unterminated_heredoc_string_literal
context = build_context
['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code| ['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code|
tokens = RubyLex.ripper_lex_without_warning(code) tokens = RubyLex.ripper_lex_without_warning(code)
string_literal = RubyLex.new.check_string_literal(tokens) string_literal = RubyLex.new(context).check_string_literal(tokens)
assert_equal('<<A', string_literal&.tok) assert_equal('<<A', string_literal&.tok)
end end
end end
@ -779,8 +780,9 @@ module TestIRB
p( p(
) )
EOC EOC
context = build_context
[reference_code, code_with_heredoc, code_with_embdoc].each do |code| [reference_code, code_with_heredoc, code_with_embdoc].each do |code|
lex = RubyLex.new lex = RubyLex.new(context)
lines = code.lines lines = code.lines
lex.instance_variable_set('@tokens', RubyLex.ripper_lex_without_warning(code)) lex.instance_variable_set('@tokens', RubyLex.ripper_lex_without_warning(code))
assert_equal 2, lex.check_corresponding_token_depth(lines, lines.size) assert_equal 2, lex.check_corresponding_token_depth(lines, lines.size)
@ -788,7 +790,7 @@ module TestIRB
end end
def test_find_prev_spaces_with_multiline_literal def test_find_prev_spaces_with_multiline_literal
lex = RubyLex.new lex = RubyLex.new(build_context)
reference_code = <<~EOC.chomp reference_code = <<~EOC.chomp
if true if true
1 1
@ -813,8 +815,9 @@ module TestIRB
world world
end end
EOC EOC
context = build_context
[reference_code, code_with_percent_string, code_with_quoted_string].each do |code| [reference_code, code_with_percent_string, code_with_quoted_string].each do |code|
lex = RubyLex.new lex = RubyLex.new(context)
lex.instance_variable_set('@tokens', RubyLex.ripper_lex_without_warning(code)) lex.instance_variable_set('@tokens', RubyLex.ripper_lex_without_warning(code))
prev_spaces = (1..code.lines.size).map { |index| lex.find_prev_spaces index } prev_spaces = (1..code.lines.size).map { |index| lex.find_prev_spaces index }
assert_equal [0, 2, 2, 2, 2, 0], prev_spaces assert_equal [0, 2, 2, 2, 2, 0], prev_spaces