From f8cab4ef8e214dc05efe8a0d4178864bce3b790a Mon Sep 17 00:00:00 2001
From: Koichi ITO <koic.ito@gmail.com>
Date: Wed, 13 Mar 2024 02:05:57 +0900
Subject: [PATCH] [ruby/prism] Fix a token incompatibility for
 `Prism::Translation::Parser::Lexer`

In practice, the `BACKTICK` is mapped either as `:tXSTRING_BEG` or `:tBACK_REF2`.
The former is used in xstrings like `` `foo` ``, while the latter is utilized as
a back reference in contexts like `` A::` ``.
This PR will make corrections to differentiate the use of `BACKTICK`.

This mistake was discovered through the investigation of xstring.txt file.
The PR will run tests from xstring.txt file except for `` `f\oo` ``, which will still fail,
hence it will be separated into xstring_with_backslash.txt file.
This separation will facilitate addressing the correction at a different time.

https://github.com/ruby/prism/commit/49ad8df40a
---
 lib/prism/translation/parser/lexer.rb         |  6 +++++-
 test/prism/fixtures/xstring.txt               |  2 --
 .../prism/fixtures/xstring_with_backslash.txt |  1 +
 test/prism/parser_test.rb                     |  2 +-
 test/prism/snapshots/xstring.txt              | 20 +++++++------------
 .../snapshots/xstring_with_backslash.txt      | 11 ++++++++++
 6 files changed, 25 insertions(+), 17 deletions(-)
 create mode 100644 test/prism/fixtures/xstring_with_backslash.txt
 create mode 100644 test/prism/snapshots/xstring_with_backslash.txt

diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb
index 054e56a4ba..92495ab3d2 100644
--- a/lib/prism/translation/parser/lexer.rb
+++ b/lib/prism/translation/parser/lexer.rb
@@ -24,7 +24,7 @@ module Prism
           AMPERSAND_DOT: :tANDDOT,
           AMPERSAND_EQUAL: :tOP_ASGN,
           BACK_REFERENCE: :tBACK_REF,
-          BACKTICK: :tBACK_REF2,
+          BACKTICK: :tXSTRING_BEG,
           BANG: :tBANG,
           BANG_EQUAL: :tNEQ,
           BANG_TILDE: :tNMATCH,
@@ -325,6 +325,10 @@ module Prism
               if !tokens.empty? && tokens.dig(-1, 0) == :kDEF
                 type = :tIDENTIFIER
               end
+            when :tXSTRING_BEG
+              if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT
+                type = :tBACK_REF2
+              end
             end
 
             tokens << [type, [value, location]]
diff --git a/test/prism/fixtures/xstring.txt b/test/prism/fixtures/xstring.txt
index 01bcc09fc5..623afd9797 100644
--- a/test/prism/fixtures/xstring.txt
+++ b/test/prism/fixtures/xstring.txt
@@ -2,6 +2,4 @@
 
 `foo #{bar} baz`
 
-`f\oo`
-
 `foo`
diff --git a/test/prism/fixtures/xstring_with_backslash.txt b/test/prism/fixtures/xstring_with_backslash.txt
new file mode 100644
index 0000000000..b51bb0f6f9
--- /dev/null
+++ b/test/prism/fixtures/xstring_with_backslash.txt
@@ -0,0 +1 @@
+`f\oo`
diff --git a/test/prism/parser_test.rb b/test/prism/parser_test.rb
index 60fe0e0f47..518cb36c42 100644
--- a/test/prism/parser_test.rb
+++ b/test/prism/parser_test.rb
@@ -77,7 +77,7 @@ module Prism
       "heredocs_nested.txt",
       "indented_file_end.txt",
       "strings.txt",
-      "xstring.txt"
+      "xstring_with_backslash.txt"
     ]
 
     Dir["*.txt", base: base].each do |name|
diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt
index 325a39afa5..0a9d075818 100644
--- a/test/prism/snapshots/xstring.txt
+++ b/test/prism/snapshots/xstring.txt
@@ -1,8 +1,8 @@
-@ ProgramNode (location: (1,0)-(7,5))
+@ ProgramNode (location: (1,0)-(5,5))
 ├── locals: []
 └── statements:
-    @ StatementsNode (location: (1,0)-(7,5))
-    └── body: (length: 4)
+    @ StatementsNode (location: (1,0)-(5,5))
+    └── body: (length: 3)
         ├── @ XStringNode (location: (1,0)-(1,7))
         │   ├── flags: ∅
         │   ├── opening_loc: (1,0)-(1,3) = "%x["
@@ -41,15 +41,9 @@
         │   │       ├── closing_loc: ∅
         │   │       └── unescaped: " baz"
         │   └── closing_loc: (3,15)-(3,16) = "`"
-        ├── @ XStringNode (location: (5,0)-(5,6))
-        │   ├── flags: ∅
-        │   ├── opening_loc: (5,0)-(5,1) = "`"
-        │   ├── content_loc: (5,1)-(5,5) = "f\\oo"
-        │   ├── closing_loc: (5,5)-(5,6) = "`"
-        │   └── unescaped: "foo"
-        └── @ XStringNode (location: (7,0)-(7,5))
+        └── @ XStringNode (location: (5,0)-(5,5))
             ├── flags: ∅
-            ├── opening_loc: (7,0)-(7,1) = "`"
-            ├── content_loc: (7,1)-(7,4) = "foo"
-            ├── closing_loc: (7,4)-(7,5) = "`"
+            ├── opening_loc: (5,0)-(5,1) = "`"
+            ├── content_loc: (5,1)-(5,4) = "foo"
+            ├── closing_loc: (5,4)-(5,5) = "`"
             └── unescaped: "foo"
diff --git a/test/prism/snapshots/xstring_with_backslash.txt b/test/prism/snapshots/xstring_with_backslash.txt
new file mode 100644
index 0000000000..7e0fa1ab5f
--- /dev/null
+++ b/test/prism/snapshots/xstring_with_backslash.txt
@@ -0,0 +1,11 @@
+@ ProgramNode (location: (1,0)-(1,6))
+├── locals: []
+└── statements:
+    @ StatementsNode (location: (1,0)-(1,6))
+    └── body: (length: 1)
+        └── @ XStringNode (location: (1,0)-(1,6))
+            ├── flags: ∅
+            ├── opening_loc: (1,0)-(1,1) = "`"
+            ├── content_loc: (1,1)-(1,5) = "f\\oo"
+            ├── closing_loc: (1,5)-(1,6) = "`"
+            └── unescaped: "foo"