[ruby/irb] Deprecate RubyLex and warn about referencing to it

(https://github.com/ruby/irb/pull/692)

`RubyLex` has always been a private component of IRB, so we should
explicitly discourage usages of it.
Also, it should be placed under the `IRB` module like other components.

https://github.com/ruby/irb/commit/069b5625f7

lib/irb/ruby-lex.rb

@@ -8,6 +8,7 @@ require "ripper"
 require "jruby" if RUBY_ENGINE == "jruby"
 require_relative "nesting_parser"
 
+module IRB
   # :stopdoc:
   class RubyLex
     ASSIGNMENT_NODE_TYPES = [
@@ -153,7 +154,7 @@ class RubyLex
 
     def check_code_state(code)
       tokens = self.class.ripper_lex_without_warning(code, context: @context)
-    opens = IRB::NestingParser.open_tokens(tokens)
+      opens = NestingParser.open_tokens(tokens)
       [tokens, opens, code_terminated?(code, tokens, opens)]
     end
 
@@ -349,7 +350,7 @@ class RubyLex
     end
 
     def process_indent_level(tokens, lines, line_index, is_newline)
-    line_results = IRB::NestingParser.parse_by_line(tokens)
+      line_results = NestingParser.parse_by_line(tokens)
       result = line_results[line_index]
       if result
         _tokens, prev_opens, next_opens, min_depth = result
@@ -484,7 +485,7 @@ class RubyLex
         if first_token && first_token.state != Ripper::EXPR_DOT
           tokens_without_last_line = tokens[0..index]
           code_without_last_line = tokens_without_last_line.map(&:tok).join
-        opens_without_last_line = IRB::NestingParser.open_tokens(tokens_without_last_line)
+          opens_without_last_line = NestingParser.open_tokens(tokens_without_last_line)
           if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line)
             return last_line_tokens.map(&:tok).join
           end
@@ -494,3 +495,7 @@ class RubyLex
     end
   end
   # :startdoc:
+end
+
+RubyLex = IRB::RubyLex
+Object.deprecate_constant(:RubyLex)

test/irb/test_nesting_parser.rb

@@ -14,7 +14,7 @@ module TestIRB
     end
 
     def parse_by_line(code)
-      IRB::NestingParser.parse_by_line(RubyLex.ripper_lex_without_warning(code))
+      IRB::NestingParser.parse_by_line(IRB::RubyLex.ripper_lex_without_warning(code))
     end
 
     def test_open_tokens
@@ -27,7 +27,7 @@ module TestIRB
                   x: "
                     #{p(1, 2, 3
       EOS
-      opens = IRB::NestingParser.open_tokens(RubyLex.ripper_lex_without_warning(code))
+      opens = IRB::NestingParser.open_tokens(IRB::RubyLex.ripper_lex_without_warning(code))
       assert_equal(%w[class def if do { " #{ (], opens.map(&:tok))
     end
 

test/irb/test_ruby_lex.rb

@@ -20,7 +20,7 @@ module TestIRB
         #{⑤&<<C|⑥
       EOC
       ripper_tokens = Ripper.tokenize(code)
-      rubylex_tokens = RubyLex.ripper_lex_without_warning(code)
+      rubylex_tokens = IRB::RubyLex.ripper_lex_without_warning(code)
       # Assert no missing part
       assert_equal(code, rubylex_tokens.map(&:tok).join)
       # Assert ripper tokens are not removed
@@ -83,7 +83,7 @@ module TestIRB
     end
 
     def test_broken_percent_literal
-      tokens = RubyLex.ripper_lex_without_warning('%wwww')
+      tokens = IRB::RubyLex.ripper_lex_without_warning('%wwww')
       pos_to_index = {}
       tokens.each_with_index { |t, i|
         assert_nil(pos_to_index[t.pos], "There is already another token in the position of #{t.inspect}.")
@@ -92,7 +92,7 @@ module TestIRB
     end
 
     def test_broken_percent_literal_in_method
-      tokens = RubyLex.ripper_lex_without_warning(<<~EOC.chomp)
+      tokens = IRB::RubyLex.ripper_lex_without_warning(<<~EOC.chomp)
         def foo
           %wwww
         end
@@ -106,7 +106,7 @@ module TestIRB
 
     def test_unterminated_code
       ['do', '<<A'].each do |code|
-        tokens = RubyLex.ripper_lex_without_warning(code)
+        tokens = IRB::RubyLex.ripper_lex_without_warning(code)
         assert_equal(code, tokens.map(&:tok).join, "Cannot reconstruct code from tokens")
         error_tokens = tokens.map(&:event).grep(/error/)
         assert_empty(error_tokens, 'Error tokens must be ignored if there is corresponding non-error token')
@@ -115,7 +115,7 @@ module TestIRB
 
     def test_unterminated_heredoc_string_literal
       ['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code|
-        tokens = RubyLex.ripper_lex_without_warning(code)
+        tokens = IRB::RubyLex.ripper_lex_without_warning(code)
         string_literal = IRB::NestingParser.open_tokens(tokens).last
         assert_equal('<<A', string_literal&.tok)
       end
@@ -150,7 +150,7 @@ module TestIRB
 
     def test_assignment_expression
       context = build_context
-      ruby_lex = RubyLex.new(context)
+      ruby_lex = IRB::RubyLex.new(context)
 
       [
         "foo = bar",
@@ -194,7 +194,7 @@ module TestIRB
 
     def test_assignment_expression_with_local_variable
       context = build_context
-      ruby_lex = RubyLex.new(context)
+      ruby_lex = IRB::RubyLex.new(context)
       code = "a /1;x=1#/"
       refute(ruby_lex.assignment_expression?(code), "#{code}: should not be an assignment expression")
       context.workspace.binding.eval('a = 1')
@@ -202,6 +202,14 @@ module TestIRB
       refute(ruby_lex.assignment_expression?(""), "empty code should not be an assignment expression")
     end
 
+    def test_initialising_the_old_top_level_ruby_lex
+      _, err = capture_output do
+        ::RubyLex.new(nil)
+      end
+
+      assert_match(/warning: constant ::RubyLex is deprecated/, err)
+    end
+
     private
 
     def build_context(local_variables = nil)
@@ -239,7 +247,7 @@ module TestIRB
     def check_state(lines, local_variables: [])
       context = build_context(local_variables)
       code = lines.map { |l| "#{l}\n" }.join # code should end with "\n"
-      ruby_lex = RubyLex.new(context)
+      ruby_lex = IRB::RubyLex.new(context)
       tokens, opens, terminated = ruby_lex.check_code_state(code)
       indent_level = ruby_lex.calc_indent_level(opens)
       continue = ruby_lex.should_continue?(tokens)