[ruby/prism] Fix an AST and token incompatibility for Prism::Translation::Parser

Fixes https://github.com/ruby/prism/pull/2506.

This PR fixes an AST and token incompatibility between Parser gem and `Prism::Translation::Parser`
for symbols quoted with line breaks.

https://github.com/ruby/prism/commit/06ab4df8cd
This commit is contained in:
Koichi ITO 2024-03-03 21:25:10 +09:00 committed by git
parent 78725f14b2
commit 0e4bfd08e5
4 changed files with 38 additions and 1 deletions

View File

@ -1528,9 +1528,23 @@ module Prism
builder.symbol([node.unescaped, srange(node.location)])
end
else
parts = if node.value.lines.one?
[builder.string_internal([node.unescaped, srange(node.value_loc)])]
else
start_offset = node.value_loc.start_offset
node.value.lines.map do |line|
end_offset = start_offset + line.length
offsets = srange_offsets(start_offset, end_offset)
start_offset = end_offset
builder.string_internal([line, offsets])
end
end
builder.symbol_compose(
token(node.opening_loc),
[builder.string_internal([node.unescaped, srange(node.value_loc)])],
parts,
token(node.closing_loc)
)
end

View File

@ -291,6 +291,16 @@ module Prism
quote = value[2] == "-" || value[2] == "~" ? value[3] : value[2]
value = "<<#{quote == "'" || quote == "\"" ? quote : "\""}"
end
when :tSTRING_CONTENT
unless (lines = token.value.lines).one?
start_offset = offset_cache[token.location.start_offset]
lines.map do |line|
end_offset = start_offset + line.length
tokens << [:tSTRING_CONTENT, [line, Range.new(source_buffer, start_offset, offset_cache[end_offset])]]
start_offset = end_offset
end
next
end
when :tSTRING_DVAR
value = nil
when :tSTRING_END

View File

@ -0,0 +1,2 @@
:"foo
bar"

View File

@ -0,0 +1,11 @@
@ ProgramNode (location: (1,0)-(2,6))
├── locals: []
└── statements:
@ StatementsNode (location: (1,0)-(2,6))
└── body: (length: 1)
└── @ SymbolNode (location: (1,0)-(2,6))
├── flags: forced_us_ascii_encoding
├── opening_loc: (1,0)-(1,2) = ":\""
├── value_loc: (1,2)-(2,5) = "foo\n bar"
├── closing_loc: (2,5)-(2,6) = "\""
└── unescaped: "foo\n bar"