[ruby/prism] Fix a token incompatibility for Prism::Translation::Parser::Lexer

In practice, the `BACKTICK` is mapped either as `:tXSTRING_BEG` or `:tBACK_REF2`.
The former is used in xstrings like `` `foo` ``, while the latter is utilized as
a back reference in contexts like `` A::` ``.
This PR will make corrections to differentiate the use of `BACKTICK`.

This mistake was discovered through the investigation of xstring.txt file.
The PR will run tests from xstring.txt file except for `` `f\oo` ``, which will still fail,
hence it will be separated into xstring_with_backslash.txt file.
This separation will facilitate addressing the correction at a different time.

https://github.com/ruby/prism/commit/49ad8df40a

lib/prism/translation/parser/lexer.rb

@@ -24,7 +24,7 @@ module Prism
           AMPERSAND_DOT: :tANDDOT,
           AMPERSAND_EQUAL: :tOP_ASGN,
           BACK_REFERENCE: :tBACK_REF,
-          BACKTICK: :tBACK_REF2,
+          BACKTICK: :tXSTRING_BEG,
           BANG: :tBANG,
           BANG_EQUAL: :tNEQ,
           BANG_TILDE: :tNMATCH,
@@ -325,6 +325,10 @@ module Prism
               if !tokens.empty? && tokens.dig(-1, 0) == :kDEF
                 type = :tIDENTIFIER
               end
+            when :tXSTRING_BEG
+              if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT
+                type = :tBACK_REF2
+              end
             end
 
             tokens << [type, [value, location]]

test/prism/fixtures/xstring.txt

@@ -2,6 +2,4 @@
 
 `foo #{bar} baz`
 
-`f\oo`
-
 `foo`

test/prism/fixtures/xstring_with_backslash.txt

@@ -0,0 +1 @@
+`f\oo`

test/prism/parser_test.rb

@@ -77,7 +77,7 @@ module Prism
       "heredocs_nested.txt",
       "indented_file_end.txt",
       "strings.txt",
-      "xstring.txt"
+      "xstring_with_backslash.txt"
     ]
 
     Dir["*.txt", base: base].each do |name|

test/prism/snapshots/xstring.txt

@@ -1,8 +1,8 @@
-@ ProgramNode (location: (1,0)-(7,5))
+@ ProgramNode (location: (1,0)-(5,5))
 ├── locals: []
 └── statements:
-    @ StatementsNode (location: (1,0)-(7,5))
+    @ StatementsNode (location: (1,0)-(5,5))
-    └── body: (length: 4)
+    └── body: (length: 3)
         ├── @ XStringNode (location: (1,0)-(1,7))
         │   ├── flags: ∅
         │   ├── opening_loc: (1,0)-(1,3) = "%x["
@@ -41,15 +41,9 @@
         │   │       ├── closing_loc: ∅
         │   │       └── unescaped: " baz"
         │   └── closing_loc: (3,15)-(3,16) = "`"
-        ├── @ XStringNode (location: (5,0)-(5,6))
+        └── @ XStringNode (location: (5,0)-(5,5))
-        │   ├── flags: ∅
-        │   ├── opening_loc: (5,0)-(5,1) = "`"
-        │   ├── content_loc: (5,1)-(5,5) = "f\\oo"
-        │   ├── closing_loc: (5,5)-(5,6) = "`"
-        │   └── unescaped: "foo"
-        └── @ XStringNode (location: (7,0)-(7,5))
             ├── flags: ∅
-            ├── opening_loc: (7,0)-(7,1) = "`"
+            ├── opening_loc: (5,0)-(5,1) = "`"
-            ├── content_loc: (7,1)-(7,4) = "foo"
+            ├── content_loc: (5,1)-(5,4) = "foo"
-            ├── closing_loc: (7,4)-(7,5) = "`"
+            ├── closing_loc: (5,4)-(5,5) = "`"
             └── unescaped: "foo"

test/prism/snapshots/xstring_with_backslash.txt

@@ -0,0 +1,11 @@
+@ ProgramNode (location: (1,0)-(1,6))
+├── locals: []
+└── statements:
+    @ StatementsNode (location: (1,0)-(1,6))
+    └── body: (length: 1)
+        └── @ XStringNode (location: (1,0)-(1,6))
+            ├── flags: ∅
+            ├── opening_loc: (1,0)-(1,1) = "`"
+            ├── content_loc: (1,1)-(1,5) = "f\\oo"
+            ├── closing_loc: (1,5)-(1,6) = "`"
+            └── unescaped: "foo"