[ruby/yarp] Move lex compat into its own file

https://github.com/ruby/yarp/commit/e90f88f21f

lib/yarp.rb

@@ -303,6 +303,8 @@ module YARP
   autoload :DesugarCompiler, "yarp/desugar_compiler"
   autoload :Dispatcher, "yarp/dispatcher"
   autoload :DSL, "yarp/dsl"
+  autoload :LexCompat, "yarp/lex_compat"
+  autoload :LexRipper, "yarp/lex_compat"
   autoload :MutationCompiler, "yarp/mutation_compiler"
   autoload :NodeInspector, "yarp/node_inspector"
   autoload :RipperCompat, "yarp/ripper_compat"
@@ -311,10 +313,25 @@ module YARP
   autoload :Serialize, "yarp/serialize"
   autoload :Visitor, "yarp/visitor"
 
-  # Marking this as private so that consumers don't see it. It makes it a little
+  # Some of these constants are not meant to be exposed, so marking them as
-  # annoying for testing since you have to const_get it to access the methods,
+  # private here.
-  # but at least this way it's clear it's not meant for consumers.
   private_constant :Debug
+  private_constant :LexCompat
+  private_constant :LexRipper
+
+  # Returns an array of tokens that closely resembles that of the Ripper lexer.
+  # The only difference is that since we don't keep track of lexer state in the
+  # same way, it's going to always return the NONE state.
+  def self.lex_compat(source, filepath = "")
+    LexCompat.new(source, filepath).result
+  end
+
+  # This lexes with the Ripper lex. It drops any space events but otherwise
+  # returns the same tokens. Raises SyntaxError if the syntax in source is
+  # invalid.
+  def self.lex_ripper(source)
+    LexRipper.new(source).result
+  end
 
   # Load the serialized AST using the source as a reference into a tree.
   def self.load(source, serialized)
@@ -322,9 +339,7 @@ module YARP
   end
 end
 
-require_relative "yarp/lex_compat"
 require_relative "yarp/node"
-
 require_relative "yarp/parse_result/comments"
 require_relative "yarp/parse_result/newlines"
 

lib/yarp/lex_compat.rb

@@ -795,21 +795,16 @@ module YARP
     end
   end
 
-  # The constant that wraps the behavior of the lexer to match Ripper's output
+  # This is a class that wraps the Ripper lexer to produce almost exactly the
-  # is an implementation detail, so we don't want it to be public.
+  # same tokens.
-  private_constant :LexCompat
+  class LexRipper
+    attr_reader :source
 
-  # Returns an array of tokens that closely resembles that of the Ripper lexer.
+    def initialize(source)
-  # The only difference is that since we don't keep track of lexer state in the
+      @source = source
-  # same way, it's going to always return the NONE state.
-  def self.lex_compat(source, filepath = "")
-    LexCompat.new(source, filepath).result
     end
 
-  # This lexes with the Ripper lex. It drops any space events but otherwise
+    def result
-  # returns the same tokens. Raises SyntaxError if the syntax in source is
-  # invalid.
-  def self.lex_ripper(source)
       previous = []
       results = []
 
@@ -840,3 +835,4 @@ module YARP
       results
     end
   end
+end