[ruby/irb] Clear all context usages in RubyLex

(https://github.com/ruby/irb/pull/684)

After this change, `RubyLex` will not interact with `Context` directly
in any way. This decoupling has a few benefits:

- It makes `RubyLex` easier to test as it no longer has a dependency on
  `Context`. We can see this from the removal of `build_context` from
  `test_ruby_lex.rb`.
- It will make `RubyLex` easier to understand as it will not be affected
  by state changes in `Context` objects.
- It allows `RubyLex` to be used in places where `Context` is not available.

https://github.com/ruby/irb/commit/d5b262a076

lib/irb.rb

@@ -436,7 +436,7 @@ module IRB
       @context = Context.new(self, workspace, input_method)
       @context.workspace.load_commands_to_main
       @signal_status = :IN_IRB
-      @scanner = RubyLex.new(@context)
+      @scanner = RubyLex.new
     end
 
     # A hook point for `debug` command's breakpoint after :IRB_EXIT as well as its clean-up
@@ -610,7 +610,7 @@ module IRB
         # Accept any single-line input for symbol aliases or commands that transform args
         return code if single_line_command?(code)
 
-        tokens, opens, terminated = @scanner.check_code_state(code)
+        tokens, opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
         return code if terminated
 
         line_offset += 1
@@ -643,7 +643,8 @@ module IRB
       if command_class
         Statement::Command.new(code, command, arg, command_class)
       else
-        Statement::Expression.new(code, @scanner.assignment_expression?(code))
+        is_assignment_expression = @scanner.assignment_expression?(code, local_variables: @context.local_variables)
+        Statement::Expression.new(code, is_assignment_expression)
       end
     end
 
@@ -656,7 +657,7 @@ module IRB
       if @context.io.respond_to?(:check_termination)
         @context.io.check_termination do |code|
           if Reline::IOGate.in_pasting?
-            rest = @scanner.check_termination_in_prev_line(code)
+            rest = @scanner.check_termination_in_prev_line(code, local_variables: @context.local_variables)
             if rest
               Reline.delete_text
               rest.bytes.reverse_each do |c|
@@ -670,7 +671,7 @@ module IRB
             # Accept any single-line input for symbol aliases or commands that transform args
             next true if single_line_command?(code)
 
-            _tokens, _opens, terminated = @scanner.check_code_state(code)
+            _tokens, _opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
             terminated
           end
         end
@@ -678,7 +679,7 @@ module IRB
       if @context.io.respond_to?(:dynamic_prompt)
         @context.io.dynamic_prompt do |lines|
           lines << '' if lines.empty?
-          tokens = RubyLex.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: @context)
+          tokens = RubyLex.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, local_variables: @context.local_variables)
           line_results = IRB::NestingParser.parse_by_line(tokens)
           tokens_until_line = []
           line_results.map.with_index do |(line_tokens, _prev_opens, next_opens, _min_depth), line_num_offset|
@@ -698,7 +699,7 @@ module IRB
           next nil if !is_newline && lines[line_index]&.byteslice(0, byte_pointer)&.match?(/\A\s*\z/)
 
           code = lines[0..line_index].map { |l| "#{l}\n" }.join
-          tokens = RubyLex.ripper_lex_without_warning(code, context: @context)
+          tokens = RubyLex.ripper_lex_without_warning(code, local_variables: @context.local_variables)
           @scanner.process_indent_level(tokens, lines, line_index, is_newline)
         end
       end

lib/irb/ruby-lex.rb

@@ -44,8 +44,7 @@ module IRB
 
     attr_reader :line_no
 
-    def initialize(context)
+    def initialize
-      @context = context
       @line_no = 1
       @prompt = nil
     end
@@ -116,9 +115,9 @@ module IRB
       interpolated
     end
 
-    def self.ripper_lex_without_warning(code, context: nil)
+    def self.ripper_lex_without_warning(code, local_variables: [])
       verbose, $VERBOSE = $VERBOSE, nil
-      lvars_code = generate_local_variables_assign_code(context&.local_variables || [])
+      lvars_code = generate_local_variables_assign_code(local_variables)
       original_code = code
       if lvars_code
         code = "#{lvars_code}\n#{code}"
@@ -152,14 +151,14 @@ module IRB
       @prompt&.call(ltype, indent_level, opens.any? || continue, @line_no + line_num_offset)
     end
 
-    def check_code_state(code)
+    def check_code_state(code, local_variables:)
-      tokens = self.class.ripper_lex_without_warning(code, context: @context)
+      tokens = self.class.ripper_lex_without_warning(code, local_variables: local_variables)
       opens = NestingParser.open_tokens(tokens)
-      [tokens, opens, code_terminated?(code, tokens, opens)]
+      [tokens, opens, code_terminated?(code, tokens, opens, local_variables: local_variables)]
     end
 
-    def code_terminated?(code, tokens, opens)
+    def code_terminated?(code, tokens, opens, local_variables:)
-      case check_code_syntax(code)
+      case check_code_syntax(code, local_variables: local_variables)
       when :unrecoverable_error
         true
       when :recoverable_error
@@ -180,7 +179,7 @@ module IRB
       @line_no += addition
     end
 
-    def assignment_expression?(code)
+    def assignment_expression?(code, local_variables:)
       # Try to parse the code and check if the last of possibly multiple
       # expressions is an assignment type.
 
@@ -190,7 +189,7 @@ module IRB
       # array of parsed expressions. The first element of each expression is the
       # expression's type.
       verbose, $VERBOSE = $VERBOSE, nil
-      code = "#{RubyLex.generate_local_variables_assign_code(@context.local_variables) || 'nil;'}\n#{code}"
+      code = "#{RubyLex.generate_local_variables_assign_code(local_variables) || 'nil;'}\n#{code}"
       # Get the last node_type of the line. drop(1) is to ignore the local_variables_assign_code part.
       node_type = Ripper.sexp(code)&.dig(1)&.drop(1)&.dig(-1, 0)
       ASSIGNMENT_NODE_TYPES.include?(node_type)
@@ -222,8 +221,8 @@ module IRB
       false
     end
 
-    def check_code_syntax(code)
+    def check_code_syntax(code, local_variables:)
-      lvars_code = RubyLex.generate_local_variables_assign_code(@context.local_variables)
+      lvars_code = RubyLex.generate_local_variables_assign_code(local_variables)
       code = "#{lvars_code}\n#{code}"
 
       begin # check if parser error are available
@@ -455,8 +454,8 @@ module IRB
       end
     end
 
-    def check_termination_in_prev_line(code)
+    def check_termination_in_prev_line(code, local_variables:)
-      tokens = self.class.ripper_lex_without_warning(code, context: @context)
+      tokens = self.class.ripper_lex_without_warning(code, local_variables: local_variables)
       past_first_newline = false
       index = tokens.rindex do |t|
         # traverse first token before last line
@@ -486,7 +485,7 @@ module IRB
           tokens_without_last_line = tokens[0..index]
           code_without_last_line = tokens_without_last_line.map(&:tok).join
           opens_without_last_line = NestingParser.open_tokens(tokens_without_last_line)
-          if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line)
+          if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line, local_variables: local_variables)
             return last_line_tokens.map(&:tok).join
           end
         end

lib/irb/source_finder.rb

@@ -43,7 +43,7 @@ module IRB
     private
 
     def find_end(file, first_line)
-      lex = RubyLex.new(@irb_context)
+      lex = RubyLex.new
       lines = File.read(file).lines[(first_line - 1)..-1]
       tokens = RubyLex.ripper_lex_without_warning(lines.join)
       prev_tokens = []
@@ -53,7 +53,7 @@ module IRB
         code = lines[0..lnum].join
         prev_tokens.concat chunk
         continue = lex.should_continue?(prev_tokens)
-        syntax = lex.check_code_syntax(code)
+        syntax = lex.check_code_syntax(code, local_variables: [])
         if !continue && syntax == :valid
           return first_line + lnum
         end

test/irb/test_irb.rb

@@ -584,7 +584,7 @@ module TestIRB
 
       def assert_indent_level(lines, expected)
         code = lines.map { |l| "#{l}\n" }.join # code should end with "\n"
-        _tokens, opens, _ = @irb.scanner.check_code_state(code)
+        _tokens, opens, _ = @irb.scanner.check_code_state(code, local_variables: [])
         indent_level = @irb.scanner.calc_indent_level(opens)
         error_message = "Calculated the wrong number of indent level for:\n #{lines.join("\n")}"
         assert_equal(expected, indent_level, error_message)

test/irb/test_ruby_lex.rb

@@ -149,8 +149,7 @@ module TestIRB
     end
 
     def test_assignment_expression
-      context = build_context
+      ruby_lex = IRB::RubyLex.new
-      ruby_lex = IRB::RubyLex.new(context)
 
       [
         "foo = bar",
@@ -173,7 +172,7 @@ module TestIRB
         "foo\nfoo = bar",
       ].each do |exp|
         assert(
-          ruby_lex.assignment_expression?(exp),
+          ruby_lex.assignment_expression?(exp, local_variables: []),
           "#{exp.inspect}: should be an assignment expression"
         )
       end
@@ -186,20 +185,18 @@ module TestIRB
         "foo = bar\nfoo",
       ].each do |exp|
         refute(
-          ruby_lex.assignment_expression?(exp),
+          ruby_lex.assignment_expression?(exp, local_variables: []),
           "#{exp.inspect}: should not be an assignment expression"
         )
       end
     end
 
     def test_assignment_expression_with_local_variable
-      context = build_context
+      ruby_lex = IRB::RubyLex.new
-      ruby_lex = IRB::RubyLex.new(context)
       code = "a /1;x=1#/"
-      refute(ruby_lex.assignment_expression?(code), "#{code}: should not be an assignment expression")
+      refute(ruby_lex.assignment_expression?(code, local_variables: []), "#{code}: should not be an assignment expression")
-      context.workspace.binding.eval('a = 1')
+      assert(ruby_lex.assignment_expression?(code, local_variables: [:a]), "#{code}: should be an assignment expression")
-      assert(ruby_lex.assignment_expression?(code), "#{code}: should be an assignment expression")
+      refute(ruby_lex.assignment_expression?("", local_variables: [:a]), "empty code should not be an assignment expression")
-      refute(ruby_lex.assignment_expression?(""), "empty code should not be an assignment expression")
     end
 
     def test_initialising_the_old_top_level_ruby_lex
@@ -211,20 +208,6 @@ module TestIRB
 
     private
 
-    def build_context(local_variables = nil)
-      IRB.init_config(nil)
-      workspace = IRB::WorkSpace.new(TOPLEVEL_BINDING.dup)
-
-      if local_variables
-        local_variables.each do |n|
-          workspace.binding.local_variable_set(n, nil)
-        end
-      end
-
-      IRB.conf[:VERBOSE] = false
-      IRB::Context.new(nil, workspace, TestInputMethod.new)
-    end
-
     def assert_indent_level(lines, expected, local_variables: [])
       indent_level, _continue, _code_block_open = check_state(lines, local_variables: local_variables)
       error_message = "Calculated the wrong number of indent level for:\n #{lines.join("\n")}"
@@ -244,10 +227,9 @@ module TestIRB
     end
 
     def check_state(lines, local_variables: [])
-      context = build_context(local_variables)
       code = lines.map { |l| "#{l}\n" }.join # code should end with "\n"
-      ruby_lex = IRB::RubyLex.new(context)
+      ruby_lex = IRB::RubyLex.new
-      tokens, opens, terminated = ruby_lex.check_code_state(code)
+      tokens, opens, terminated = ruby_lex.check_code_state(code, local_variables: local_variables)
       indent_level = ruby_lex.calc_indent_level(opens)
       continue = ruby_lex.should_continue?(tokens)
       [indent_level, continue, !terminated]