1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ruby/prism] Add node ids to nodes

Browse Source 
https://github.com/ruby/prism/commit/bf16ade7f9
This commit is contained in:
Kevin Newton 2024-07-03 10:42:56 -04:00
parent 39dcfe26ee
commit 2bf9ae3fa1
10 changed files with 449 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lib/prism/node_ext.rb
View File

@ -72,6 +72,7 @@ module Prism
def to_interpolated
InterpolatedStringNode.new(
source,
-1,
location,
frozen? ? InterpolatedStringNodeFlags::FROZEN : 0,
opening_loc,
@ -89,10 +90,11 @@ module Prism
def to_interpolated
InterpolatedXStringNode.new(
source,
-1,
location,
flags,
opening_loc,
[StringNode.new(source, content_loc, 0, nil, content_loc, nil, unescaped)],
[StringNode.new(source, node_id, content_loc, 0, nil, content_loc, nil, unescaped)],
closing_loc
)
end
@ -119,9 +121,9 @@ module Prism
deprecated("value", "numerator", "denominator")
if denominator == 1
IntegerNode.new(source, location.chop, flags, numerator)
IntegerNode.new(source, -1, location.chop, flags, numerator)
else
FloatNode.new(source, location.chop, 0, numerator.to_f / denominator)
FloatNode.new(source, -1, location.chop, 0, numerator.to_f / denominator)
end
end
end
@ -199,7 +201,12 @@ module Prism
# continue to supply that API.
def child
deprecated("name", "name_loc")
name ? ConstantReadNode.new(source, name_loc, 0, name) : MissingNode.new(source, location, 0)
if name
ConstantReadNode.new(source, -1, name_loc, 0, name)
else
MissingNode.new(source, -1, location, 0)
end
end
end
@ -235,7 +242,12 @@ module Prism
# continue to supply that API.
def child
deprecated("name", "name_loc")
name ? ConstantReadNode.new(source, name_loc, 0, name) : MissingNode.new(source, location, 0)
if name
ConstantReadNode.new(source, -1, name_loc, 0, name)
else
MissingNode.new(source, -1, location, 0)
end
end
end

7
prism/parser.h
View File

@ -625,6 +625,13 @@ typedef uint32_t pm_state_stack_t;
* it's considering.
*/
struct pm_parser {
/**
* The next node identifier that will be assigned. This is a unique
* identifier used to track nodes such that the syntax tree can be dropped
* but the node can be found through another parse.
*/
uint32_t node_id;
/** The current state of the lexer. */
pm_lex_state_t lex_state;

536
prism/prism.c
View File

File diff suppressed because it is too large Load Diff

13
prism/templates/ext/prism/api_node.c.erb
View File

@ -173,17 +173,20 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
<%- if node.fields.any? { |field| ![Prism::Template::NodeField, Prism::Template::OptionalNodeField].include?(field.class) } -%>
pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node;
<%- end -%>
VALUE argv[<%= node.fields.length + 3 %>];
VALUE argv[<%= node.fields.length + 4 %>];
// source
argv[0] = source;
// node_id
argv[1] = ULONG2NUM(node->node_id);
// location
argv[1] = pm_location_new(parser, node->location.start, node->location.end);
argv[2] = pm_location_new(parser, node->location.start, node->location.end);
// flags
argv[2] = ULONG2NUM(node->flags);
<%- node.fields.each.with_index(3) do |field, index| -%>
argv[3] = ULONG2NUM(node->flags);
<%- node.fields.each.with_index(4) do |field, index| -%>
// <%= field.name %>
<%- case field -%>
@ -235,7 +238,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
<%- end -%>
<%- end -%>
rb_ary_push(value_stack, rb_class_new_instance(<%= node.fields.length + 3 %>, argv, rb_cPrism<%= node.name %>));
rb_ary_push(value_stack, rb_class_new_instance(<%= node.fields.length + 4 %>, argv, rb_cPrism<%= node.name %>));
break;
}
<%- end -%>

6
prism/templates/include/prism/ast.h.erb
View File

@ -136,6 +136,12 @@ typedef struct pm_node {
*/
pm_node_flags_t flags;
/**
* The unique identifier for this node, which is deterministic based on the
* source. It is used to identify unique nodes across parses.
*/
uint32_t node_id;
/**
* This is the location of the node in the source. It's a range of bytes
* containing a start and an end.

6
prism/templates/lib/prism/dsl.rb.erb
View File

@ -6,11 +6,13 @@ module Prism
#
# Prism::ArrayNode.new(
# source,
# 0,
# Prism::Location.new(source, 0, 3),
# 0,
# [
# Prism::IntegerNode.new(
# source,
# 0,
# Prism::Location.new(source, 1, 1),
# Prism::IntegerBaseFlags::DECIMAL,
# 1
@ -65,7 +67,7 @@ module Prism
<%- nodes.each do |node| -%>
# Create a new <%= node.name %> node.
def <%= node.human %>(<%= ["source: default_source", "location: default_location", "flags: 0", *node.fields.map { |field|
def <%= node.human %>(<%= ["source: default_source", "node_id: 0", "location: default_location", "flags: 0", *node.fields.map { |field|
case field
when Prism::Template::NodeField, Prism::Template::ConstantField
"#{field.name}: default_node(source, location)"
@ -83,7 +85,7 @@ module Prism
raise
end
}].join(", ") %>)
<%= node.name %>.new(<%= ["source", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
<%= node.name %>.new(<%= ["source", "node_id", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
end
<%- end -%>
<%- flags.each do |flag| -%>

19
prism/templates/lib/prism/node.rb.erb
View File

@ -6,6 +6,12 @@ module Prism
attr_reader :source
private :source
# A unique identifier for this node. This is used in a very specific
# use case where you want to keep around a reference to a node without
# having to keep around the syntax tree in memory. This unique identifier
# will be consistent across multiple parses of the same source code.
attr_reader :node_id
# A Location instance that represents the location of this node in the
# source.
def location
@ -212,8 +218,9 @@ module Prism
<%- end -%>
class <%= node.name -%> < Node
# Initialize a new <%= node.name %> node.
def initialize(<%= ["source", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
def initialize(<%= ["source", "node_id", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
@source = source
@node_id = node_id
@location = location
@flags = flags
<%- node.fields.each do |field| -%>
@ -274,17 +281,17 @@ module Prism
}.compact.join(", ") %>] #: Array[Prism::node | Location]
end
# def copy: (<%= (["?location: Location", "?flags: Integer"] + node.fields.map { |field| "?#{field.name}: #{field.rbs_class}" }).join(", ") %>) -> <%= node.name %>
def copy(<%= (["location", "flags"] + node.fields.map(&:name)).map { |field| "#{field}: self.#{field}" }.join(", ") %>)
<%= node.name %>.new(<%= ["source", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
# def copy: (<%= (["?node_id: Integer", "?location: Location", "?flags: Integer"] + node.fields.map { |field| "?#{field.name}: #{field.rbs_class}" }).join(", ") %>) -> <%= node.name %>
def copy(<%= (["node_id", "location", "flags"] + node.fields.map(&:name)).map { |field| "#{field}: self.#{field}" }.join(", ") %>)
<%= node.name %>.new(<%= ["source", "node_id", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
end
# def deconstruct: () -> Array[nil | Node]
alias deconstruct child_nodes
# def deconstruct_keys: (Array[Symbol] keys) -> { <%= (node.fields.map { |field| "#{field.name}: #{field.rbs_class}" } + ["location: Location"]).join(", ") %> }
# def deconstruct_keys: (Array[Symbol] keys) -> { <%= (["node_id: Integer", "location: Location"] + node.fields.map { |field| "#{field.name}: #{field.rbs_class}" }).join(", ") %> }
def deconstruct_keys(keys)
{ <%= (node.fields.map { |field| "#{field.name}: #{field.name}" } + ["location: location"]).join(", ") %> }
{ <%= (["node_id: node_id", "location: location"] + node.fields.map { |field| "#{field.name}: #{field.name}" }).join(", ") %> }
end
<%- if (node_flags = node.flags) -%>
<%- node_flags.values.each do |value| -%>

6
prism/templates/lib/prism/serialize.rb.erb
View File

@ -322,6 +322,7 @@ module Prism
if RUBY_ENGINE == "ruby"
def load_node
type = io.getbyte
node_id = load_varuint
location = load_location
case type
@ -330,7 +331,7 @@ module Prism
<%- if node.needs_serialized_length? -%>
load_uint32
<%- end -%>
<%= node.name %>.new(<%= ["source", "location", "load_varuint", *node.fields.map { |field|
<%= node.name %>.new(<%= ["source", "node_id", "location", "load_varuint", *node.fields.map { |field|
case field
when Prism::Template::NodeField then "load_node"
when Prism::Template::OptionalNodeField then "load_optional_node"
@ -362,11 +363,12 @@ module Prism
nil,
<%- nodes.each do |node| -%>
-> {
node_id = load_varuint
location = load_location
<%- if node.needs_serialized_length? -%>
load_uint32
<%- end -%>
<%= node.name %>.new(<%= ["source", "location", "load_varuint", *node.fields.map { |field|
<%= node.name %>.new(<%= ["source", "node_id", "location", "load_varuint", *node.fields.map { |field|
case field
when Prism::Template::NodeField then "load_node"
when Prism::Template::OptionalNodeField then "load_optional_node"

1
prism/templates/src/serialize.c.erb
View File

@ -74,6 +74,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
size_t offset = buffer->length;
pm_buffer_append_varuint(buffer, node->node_id);
pm_serialize_location(parser, &node->location, buffer);
switch (PM_NODE_TYPE(node)) {

27
test/prism/result/node_id_test.rb Normal file
View File

@ -0,0 +1,27 @@
# frozen_string_literal: true
require_relative "../test_helper"
module Prism
class NodeIdTest < TestCase
Fixture.each do |fixture|
define_method(fixture.test_name) { assert_node_ids(fixture.read) }
end
private
def assert_node_ids(source)
queue = [Prism.parse(source).value]
node_ids = []
while (node = queue.shift)
node_ids << node.node_id
queue.concat(node.compact_child_nodes)
end
node_ids.sort!
refute_includes node_ids, 0
assert_equal node_ids, node_ids.uniq
end
end
end