[ruby/prism] Add node ids to nodes

https://github.com/ruby/prism/commit/bf16ade7f9

lib/prism/node_ext.rb

@@ -72,6 +72,7 @@ module Prism
     def to_interpolated
       InterpolatedStringNode.new(
         source,
+        -1,
         location,
         frozen? ? InterpolatedStringNodeFlags::FROZEN : 0,
         opening_loc,
@@ -89,10 +90,11 @@ module Prism
     def to_interpolated
       InterpolatedXStringNode.new(
         source,
+        -1,
         location,
         flags,
         opening_loc,
-        [StringNode.new(source, content_loc, 0, nil, content_loc, nil, unescaped)],
+        [StringNode.new(source, node_id, content_loc, 0, nil, content_loc, nil, unescaped)],
         closing_loc
       )
     end
@@ -119,9 +121,9 @@ module Prism
       deprecated("value", "numerator", "denominator")
 
       if denominator == 1
-        IntegerNode.new(source, location.chop, flags, numerator)
+        IntegerNode.new(source, -1, location.chop, flags, numerator)
       else
-        FloatNode.new(source, location.chop, 0, numerator.to_f / denominator)
+        FloatNode.new(source, -1, location.chop, 0, numerator.to_f / denominator)
       end
     end
   end
@@ -199,7 +201,12 @@ module Prism
     # continue to supply that API.
     def child
       deprecated("name", "name_loc")
-      name ? ConstantReadNode.new(source, name_loc, 0, name) : MissingNode.new(source, location, 0)
+
+      if name
+        ConstantReadNode.new(source, -1, name_loc, 0, name)
+      else
+        MissingNode.new(source, -1, location, 0)
+      end
     end
   end
 
@@ -235,7 +242,12 @@ module Prism
     # continue to supply that API.
     def child
       deprecated("name", "name_loc")
-      name ? ConstantReadNode.new(source, name_loc, 0, name) : MissingNode.new(source, location, 0)
+
+      if name
+        ConstantReadNode.new(source, -1, name_loc, 0, name)
+      else
+        MissingNode.new(source, -1, location, 0)
+      end
     end
   end
 

prism/parser.h

@@ -625,6 +625,13 @@ typedef uint32_t pm_state_stack_t;
  * it's considering.
  */
 struct pm_parser {
+    /**
+     * The next node identifier that will be assigned. This is a unique
+     * identifier used to track nodes such that the syntax tree can be dropped
+     * but the node can be found through another parse.
+     */
+    uint32_t node_id;
+
     /** The current state of the lexer. */
     pm_lex_state_t lex_state;
 

prism/templates/ext/prism/api_node.c.erb

@@ -173,17 +173,20 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
                     <%- if node.fields.any? { |field| ![Prism::Template::NodeField, Prism::Template::OptionalNodeField].include?(field.class) } -%>
                     pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node;
                     <%- end -%>
-                    VALUE argv[<%= node.fields.length + 3 %>];
+                    VALUE argv[<%= node.fields.length + 4 %>];
 
                     // source
                     argv[0] = source;
 
+                    // node_id
+                    argv[1] = ULONG2NUM(node->node_id);
+
                     // location
-                    argv[1] = pm_location_new(parser, node->location.start, node->location.end);
+                    argv[2] = pm_location_new(parser, node->location.start, node->location.end);
 
                     // flags
-                    argv[2] = ULONG2NUM(node->flags);
+                    argv[3] = ULONG2NUM(node->flags);
-                    <%- node.fields.each.with_index(3) do |field, index| -%>
+                    <%- node.fields.each.with_index(4) do |field, index| -%>
 
                     // <%= field.name %>
                     <%- case field -%>
@@ -235,7 +238,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
                     <%- end -%>
                     <%- end -%>
 
-                    rb_ary_push(value_stack, rb_class_new_instance(<%= node.fields.length + 3 %>, argv, rb_cPrism<%= node.name %>));
+                    rb_ary_push(value_stack, rb_class_new_instance(<%= node.fields.length + 4 %>, argv, rb_cPrism<%= node.name %>));
                     break;
                 }
                 <%- end -%>

prism/templates/include/prism/ast.h.erb

@@ -136,6 +136,12 @@ typedef struct pm_node {
      */
     pm_node_flags_t flags;
 
+    /**
+     * The unique identifier for this node, which is deterministic based on the
+     * source. It is used to identify unique nodes across parses.
+     */
+    uint32_t node_id;
+
     /**
      * This is the location of the node in the source. It's a range of bytes
      * containing a start and an end.

prism/templates/lib/prism/dsl.rb.erb

@@ -6,11 +6,13 @@ module Prism
   #
   #     Prism::ArrayNode.new(
   #       source,
+  #       0,
   #       Prism::Location.new(source, 0, 3),
   #       0,
   #       [
   #         Prism::IntegerNode.new(
   #           source,
+  #           0,
   #           Prism::Location.new(source, 1, 1),
   #           Prism::IntegerBaseFlags::DECIMAL,
   #           1
@@ -65,7 +67,7 @@ module Prism
     <%- nodes.each do |node| -%>
 
     # Create a new <%= node.name %> node.
-    def <%= node.human %>(<%= ["source: default_source", "location: default_location", "flags: 0", *node.fields.map { |field|
+    def <%= node.human %>(<%= ["source: default_source", "node_id: 0", "location: default_location", "flags: 0", *node.fields.map { |field|
       case field
       when Prism::Template::NodeField, Prism::Template::ConstantField
         "#{field.name}: default_node(source, location)"
@@ -83,7 +85,7 @@ module Prism
         raise
       end
     }].join(", ") %>)
-      <%= node.name %>.new(<%= ["source", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
+      <%= node.name %>.new(<%= ["source", "node_id", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
     end
     <%- end -%>
     <%- flags.each do |flag| -%>

prism/templates/lib/prism/node.rb.erb

@@ -6,6 +6,12 @@ module Prism
     attr_reader :source
     private :source
 
+    # A unique identifier for this node. This is used in a very specific
+    # use case where you want to keep around a reference to a node without
+    # having to keep around the syntax tree in memory. This unique identifier
+    # will be consistent across multiple parses of the same source code.
+    attr_reader :node_id
+
     # A Location instance that represents the location of this node in the
     # source.
     def location
@@ -212,8 +218,9 @@ module Prism
   <%- end -%>
   class <%= node.name -%> < Node
     # Initialize a new <%= node.name %> node.
-    def initialize(<%= ["source", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
+    def initialize(<%= ["source", "node_id", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
       @source = source
+      @node_id = node_id
       @location = location
       @flags = flags
       <%- node.fields.each do |field| -%>
@@ -274,17 +281,17 @@ module Prism
       }.compact.join(", ") %>] #: Array[Prism::node | Location]
     end
 
-    # def copy: (<%= (["?location: Location", "?flags: Integer"] + node.fields.map { |field| "?#{field.name}: #{field.rbs_class}" }).join(", ") %>) -> <%= node.name %>
+    # def copy: (<%= (["?node_id: Integer", "?location: Location", "?flags: Integer"] + node.fields.map { |field| "?#{field.name}: #{field.rbs_class}" }).join(", ") %>) -> <%= node.name %>
-    def copy(<%= (["location", "flags"] + node.fields.map(&:name)).map { |field| "#{field}: self.#{field}" }.join(", ") %>)
+    def copy(<%= (["node_id", "location", "flags"] + node.fields.map(&:name)).map { |field| "#{field}: self.#{field}" }.join(", ") %>)
-      <%= node.name %>.new(<%= ["source", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
+      <%= node.name %>.new(<%= ["source", "node_id", "location", "flags", *node.fields.map(&:name)].join(", ") %>)
     end
 
     # def deconstruct: () -> Array[nil | Node]
     alias deconstruct child_nodes
 
-    # def deconstruct_keys: (Array[Symbol] keys) -> { <%= (node.fields.map { |field| "#{field.name}: #{field.rbs_class}" } + ["location: Location"]).join(", ") %> }
+    # def deconstruct_keys: (Array[Symbol] keys) -> { <%= (["node_id: Integer", "location: Location"] + node.fields.map { |field| "#{field.name}: #{field.rbs_class}" }).join(", ") %> }
     def deconstruct_keys(keys)
-      { <%= (node.fields.map { |field| "#{field.name}: #{field.name}" } + ["location: location"]).join(", ") %> }
+      { <%= (["node_id: node_id", "location: location"] + node.fields.map { |field| "#{field.name}: #{field.name}" }).join(", ") %> }
     end
     <%- if (node_flags = node.flags) -%>
     <%- node_flags.values.each do |value| -%>

prism/templates/lib/prism/serialize.rb.erb

@@ -322,6 +322,7 @@ module Prism
       if RUBY_ENGINE == "ruby"
         def load_node
           type = io.getbyte
+          node_id = load_varuint
           location = load_location
 
           case type
@@ -330,7 +331,7 @@ module Prism
             <%- if node.needs_serialized_length? -%>
             load_uint32
             <%- end -%>
-            <%= node.name %>.new(<%= ["source", "location", "load_varuint", *node.fields.map { |field|
+            <%= node.name %>.new(<%= ["source", "node_id", "location", "load_varuint", *node.fields.map { |field|
               case field
               when Prism::Template::NodeField then "load_node"
               when Prism::Template::OptionalNodeField then "load_optional_node"
@@ -362,11 +363,12 @@ module Prism
             nil,
             <%- nodes.each do |node| -%>
             -> {
+              node_id = load_varuint
               location = load_location
               <%- if node.needs_serialized_length? -%>
               load_uint32
               <%- end -%>
-              <%= node.name %>.new(<%= ["source", "location", "load_varuint", *node.fields.map { |field|
+              <%= node.name %>.new(<%= ["source", "node_id", "location", "load_varuint", *node.fields.map { |field|
                 case field
                 when Prism::Template::NodeField then "load_node"
                 when Prism::Template::OptionalNodeField then "load_optional_node"

prism/templates/src/serialize.c.erb

@@ -74,6 +74,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
 
     size_t offset = buffer->length;
 
+    pm_buffer_append_varuint(buffer, node->node_id);
     pm_serialize_location(parser, &node->location, buffer);
 
     switch (PM_NODE_TYPE(node)) {

test/prism/result/node_id_test.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require_relative "../test_helper"
+
+module Prism
+  class NodeIdTest < TestCase
+    Fixture.each do |fixture|
+      define_method(fixture.test_name) { assert_node_ids(fixture.read) }
+    end
+
+    private
+
+    def assert_node_ids(source)
+      queue = [Prism.parse(source).value]
+      node_ids = []
+
+      while (node = queue.shift)
+        node_ids << node.node_id
+        queue.concat(node.compact_child_nodes)
+      end
+
+      node_ids.sort!
+      refute_includes node_ids, 0
+      assert_equal node_ids, node_ids.uniq
+    end
+  end
+end