[ruby/irb] Move assignment check to RubyLex

(https://github.com/ruby/irb/pull/670)

Since assignment check relies on tokenization with `Ripper`, it feels like
the responsibility of `RubyLex`. `Irb#eval_input` should simply get the result
when calling `each_top_level_statement` on `RubyLex`.

https://github.com/ruby/irb/commit/89d1adb3fd

lib/irb.rb

@@ -429,30 +429,6 @@ module IRB
   end
 
   class Irb
-    ASSIGNMENT_NODE_TYPES = [
-      # Local, instance, global, class, constant, instance, and index assignment:
-      #   "foo = bar",
-      #   "@foo = bar",
-      #   "$foo = bar",
-      #   "@@foo = bar",
-      #   "::Foo = bar",
-      #   "a::Foo = bar",
-      #   "Foo = bar"
-      #   "foo.bar = 1"
-      #   "foo[1] = bar"
-      :assign,
-
-      # Operation assignment:
-      #   "foo += bar"
-      #   "foo -= bar"
-      #   "foo ||= bar"
-      #   "foo &&= bar"
-      :opassign,
-
-      # Multiple assignment:
-      #   "foo, bar = 1, 2
-      :massign,
-    ]
     # Note: instance and index assignment expressions could also be written like:
     # "foo.bar=(1)" and "foo.[]=(1, bar)", when expressed that way, the former
     # be parsed as :assign and echo will be suppressed, but the latter is
@@ -563,11 +539,9 @@ module IRB
 
       @scanner.configure_io(@context.io)
 
-      @scanner.each_top_level_statement do |line, line_no|
+      @scanner.each_top_level_statement do |line, line_no, is_assignment|
         signal_status(:IN_EVAL) do
           begin
-            # Assignment expression check should be done before evaluate_line to handle code like `a /2#/ if false; a = 1`
-            is_assignment = assignment_expression?(line)
             evaluate_line(line, line_no)
 
             # Don't echo if the line ends with a semicolon
@@ -876,24 +850,6 @@ module IRB
       end
       format("#<%s: %s>", self.class, ary.join(", "))
     end
-
-    def assignment_expression?(line)
-      # Try to parse the line and check if the last of possibly multiple
-      # expressions is an assignment type.
-
-      # If the expression is invalid, Ripper.sexp should return nil which will
-      # result in false being returned. Any valid expression should return an
-      # s-expression where the second element of the top level array is an
-      # array of parsed expressions. The first element of each expression is the
-      # expression's type.
-      verbose, $VERBOSE = $VERBOSE, nil
-      code = "#{RubyLex.generate_local_variables_assign_code(@context.local_variables) || 'nil;'}\n#{line}"
-      # Get the last node_type of the line. drop(1) is to ignore the local_variables_assign_code part.
-      node_type = Ripper.sexp(code)&.dig(1)&.drop(1)&.dig(-1, 0)
-      ASSIGNMENT_NODE_TYPES.include?(node_type)
-    ensure
-      $VERBOSE = verbose
-    end
   end
 
   def @CONF.inspect

lib/irb/ruby-lex.rb

@@ -10,6 +10,30 @@ require_relative "nesting_parser"
 
 # :stopdoc:
 class RubyLex
+  ASSIGNMENT_NODE_TYPES = [
+    # Local, instance, global, class, constant, instance, and index assignment:
+    #   "foo = bar",
+    #   "@foo = bar",
+    #   "$foo = bar",
+    #   "@@foo = bar",
+    #   "::Foo = bar",
+    #   "a::Foo = bar",
+    #   "Foo = bar"
+    #   "foo.bar = 1"
+    #   "foo[1] = bar"
+    :assign,
+
+    # Operation assignment:
+    #   "foo += bar"
+    #   "foo -= bar"
+    #   "foo ||= bar"
+    #   "foo &&= bar"
+    :opassign,
+
+    # Multiple assignment:
+    #   "foo, bar = 1, 2
+    :massign,
+  ]
 
   class TerminateLineInput < StandardError
     def initialize
@@ -248,13 +272,31 @@ class RubyLex
 
       if code != "\n"
         code.force_encoding(@io.encoding)
-        yield code, @line_no
+        yield code, @line_no, assignment_expression?(code)
       end
       @line_no += code.count("\n")
     rescue TerminateLineInput
     end
   end
 
+  def assignment_expression?(line)
+    # Try to parse the line and check if the last of possibly multiple
+    # expressions is an assignment type.
+
+    # If the expression is invalid, Ripper.sexp should return nil which will
+    # result in false being returned. Any valid expression should return an
+    # s-expression where the second element of the top level array is an
+    # array of parsed expressions. The first element of each expression is the
+    # expression's type.
+    verbose, $VERBOSE = $VERBOSE, nil
+    code = "#{RubyLex.generate_local_variables_assign_code(@context.local_variables) || 'nil;'}\n#{line}"
+    # Get the last node_type of the line. drop(1) is to ignore the local_variables_assign_code part.
+    node_type = Ripper.sexp(code)&.dig(1)&.drop(1)&.dig(-1, 0)
+    ASSIGNMENT_NODE_TYPES.include?(node_type)
+  ensure
+    $VERBOSE = verbose
+  end
+
   def should_continue?(tokens)
     # Look at the last token and check if IRB need to continue reading next line.
     # Example code that should continue: `a\` `a +` `a.`

test/irb/test_context.rb

@@ -202,59 +202,6 @@ module TestIRB
       assert_equal(true, @context.use_autocomplete?)
     end
 
-    def test_assignment_expression
-      input = TestInputMethod.new
-      irb = IRB::Irb.new(IRB::WorkSpace.new(Object.new), input)
-      [
-        "foo = bar",
-        "@foo = bar",
-        "$foo = bar",
-        "@@foo = bar",
-        "::Foo = bar",
-        "a::Foo = bar",
-        "Foo = bar",
-        "foo.bar = 1",
-        "foo[1] = bar",
-        "foo += bar",
-        "foo -= bar",
-        "foo ||= bar",
-        "foo &&= bar",
-        "foo, bar = 1, 2",
-        "foo.bar=(1)",
-        "foo; foo = bar",
-        "foo; foo = bar; ;\n ;",
-        "foo\nfoo = bar",
-      ].each do |exp|
-        assert(
-          irb.assignment_expression?(exp),
-          "#{exp.inspect}: should be an assignment expression"
-        )
-      end
-
-      [
-        "foo",
-        "foo.bar",
-        "foo[0]",
-        "foo = bar; foo",
-        "foo = bar\nfoo",
-      ].each do |exp|
-        refute(
-          irb.assignment_expression?(exp),
-          "#{exp.inspect}: should not be an assignment expression"
-        )
-      end
-    end
-
-    def test_assignment_expression_with_local_variable
-      input = TestInputMethod.new
-      irb = IRB::Irb.new(IRB::WorkSpace.new(Object.new), input)
-      code = "a /1;x=1#/"
-      refute(irb.assignment_expression?(code), "#{code}: should not be an assignment expression")
-      irb.context.workspace.binding.eval('a = 1')
-      assert(irb.assignment_expression?(code), "#{code}: should be an assignment expression")
-      refute(irb.assignment_expression?(""), "empty code should not be an assignment expression")
-    end
-
     def test_echo_on_assignment
       input = TestInputMethod.new([
         "a = 1\n",

test/irb/test_ruby_lex.rb

@@ -813,6 +813,60 @@ module TestIRB
       assert_indent_level(code_with_embdoc.lines, expected)
     end
 
+    def test_assignment_expression
+      context = build_context
+      ruby_lex = RubyLex.new(context)
+
+      [
+        "foo = bar",
+        "@foo = bar",
+        "$foo = bar",
+        "@@foo = bar",
+        "::Foo = bar",
+        "a::Foo = bar",
+        "Foo = bar",
+        "foo.bar = 1",
+        "foo[1] = bar",
+        "foo += bar",
+        "foo -= bar",
+        "foo ||= bar",
+        "foo &&= bar",
+        "foo, bar = 1, 2",
+        "foo.bar=(1)",
+        "foo; foo = bar",
+        "foo; foo = bar; ;\n ;",
+        "foo\nfoo = bar",
+      ].each do |exp|
+        assert(
+          ruby_lex.assignment_expression?(exp),
+          "#{exp.inspect}: should be an assignment expression"
+        )
+      end
+
+      [
+        "foo",
+        "foo.bar",
+        "foo[0]",
+        "foo = bar; foo",
+        "foo = bar\nfoo",
+      ].each do |exp|
+        refute(
+          ruby_lex.assignment_expression?(exp),
+          "#{exp.inspect}: should not be an assignment expression"
+        )
+      end
+    end
+
+    def test_assignment_expression_with_local_variable
+      context = build_context
+      ruby_lex = RubyLex.new(context)
+      code = "a /1;x=1#/"
+      refute(ruby_lex.assignment_expression?(code), "#{code}: should not be an assignment expression")
+      context.workspace.binding.eval('a = 1')
+      assert(ruby_lex.assignment_expression?(code), "#{code}: should be an assignment expression")
+      refute(ruby_lex.assignment_expression?(""), "empty code should not be an assignment expression")
+    end
+
     private
 
     def build_context(local_variables = nil)