[ruby/yarp] Only keep semantic fields in Java, i.e. skip location fields

* Add $YARP_SERIALIZE_ONLY_SEMANTICS_FIELDS to control where to serialize location fields at templating time,
  this way there is no overhead for either case and nothing to check at runtime.
* Add a byte in the header to indicate whether location fields are included as expected.
* Fixes https://github.com/ruby/yarp/issues/807
* Simplify the build-java CI job now that the FFI backend is available so JRuby can serialize.
* Support keeping some location fields which are still needed until there is a replacement

https://github.com/ruby/yarp/commit/fc5cf2df12

.gitignore

@@ -1,267 +0,0 @@
-*-*-*.def
-*-*-*.exp
-*-*-*.lib
-*.a
-*.bak
-*.bc
-*.bundle
-*.dSYM
-*.dmyh
-*.dylib
-*.elc
-*.i
-*.ii
-*.inc
-*.log
-*.o
-*.o.tmp
-*.obj
-*.old
-*.orig
-*.pch
-*.pdb
-*.rbinc
-*.rej
-*.s
-*.sav
-*.sl
-*.so
-*.so.*
-*.swp
-*.yarb
-*~
-.*.list
-.*.time
-.DS_Store
-.bundle
-.ccmalloc
-.ext
-.pc
-.ppack
-.svn
-.time
-.ruby-version
-Makefile
-cygruby*.def
-extconf.h
-y.output
-y.tab.c
-*.gcda
-*.gcno
-*.gcov
-*.vscode
-!misc/.vscode
-lcov*.info
-
-# /
-/*-fake.rb
-/*.dll
-/*.exe
-/*.res
-/*.pc
-/*.rc
-/*_prelude.c
-/.downloaded-cache
-/.top-enc.mk
-/build*/
-/COPYING.LIB
-/ChangeLog
-/Doxyfile
-/GNUmakefile
-/README.atheos
-/README.fat-patch
-/README.v6
-/TAGS
-/archive
-/autom4te*.cache
-/automake
-/benchmark/benchmark-driver
-/beos
-/bmlog-*
-/breakpoints.gdb
-/config.cache
-/config.h
-/config.h.in
-/config.status
-/config.status.lineno
-/configure
-/coverage/simplecov
-/coverage/simplecov-html
-/coverage/doclie
-/coverage/.last_run.json
-/coverage/.resultset.json*
-/coverage/assets
-/coverage/index.html
-/doc/capi
-/enc.mk
-/encdb.h
-/exts.mk
-/goruby
-/id.[ch]
-/largefile.h
-/lcov-c-out
-/lcov-rb-out
-/lcov-out
-/lex.c
-/libruby*.*
-/miniprelude.c
-/miniruby
-/newdate.rb
-/newline.c
-/newver.rb
-/parse.c
-/parse.h
-/patches
-/patches-master
-/pitest.rb
-/ppack
-/prelude.c
-/preview
-/probes.dmyh
-/probes.h
-/rbconfig.rb
-/rename2.h
-/repack
-/revision.h
-/revision.tmp
-/ripper.tmp.y
-/riscos
-/rubicon
-/ruby
-/ruby-runner
-/ruby-runner.h
-/ruby-man.rd.gz
-/rubyspec_temp
-/run.gdb
-/sizes.c
-/static-ruby
-/test.rb
-/test-coverage.dat
-/tmp
-/transdb.h
-/uncommon.mk
-/verconf.h
-/verconf.mk
-/web
-/yasmdata.rb
-
-# /bin/
-
-/bin/*.exe
-/bin/*.dll
-/bin/goruby
-/bin/ruby
-
-# /benchmark/
-/benchmark/bm_require.data
-/benchmark/bmx_*.rb
-/benchmark/fasta.output.*
-/benchmark/wc.input
-
-/enc/*.def
-/enc/*.exp
-/enc/*.lib
-/enc/jis/props.h
-/enc/unicode/data
-
-# /coroutine/
-!/coroutine/**/*.s
-
-# /enc/trans/
-/enc/trans/*.c
-/enc/trans/*.def
-/enc/trans/*.exp
-/enc/trans/*.lib
-
-# /exe/
-/exe/goruby
-/exe/ruby
-
-# /ext/
-/ext/extinit.c
-/ext/configure-ext.mk
-/ext/*/exts.mk
-
-# /ext/-test-/cxxanyargs
-/ext/-test-/cxxanyargs/failure*.failed
-
-# /ext/-test-/win32/dln/
-/ext/-test-/win32/dln/dlntest.dll
-/ext/-test-/win32/dln/dlntest.exp
-/ext/-test-/win32/dln/dlntest.lib
-
-# /ext/-test-/gems
-/ext/-test-/gems
-
-# /ext/etc/
-/ext/etc/constdefs.h
-
-# /ext/fiddle/
-/ext/fiddle/libffi-*
-
-# /ext/rbconfig/
-/ext/rbconfig/sizeof/sizes.c
-/ext/rbconfig/sizeof/limits.c
-
-# /ext/ripper/
-/ext/ripper/eventids1.c
-/ext/ripper/eventids1.h
-/ext/ripper/.eventids2-check
-/ext/ripper/eventids2table.c
-/ext/ripper/ripper_init.c
-/ext/ripper/ripper.*
-/ext/ripper/ids1
-/ext/ripper/ids2
-
-# /ext/socket/
-/ext/socket/constants.h
-/ext/socket/constdefs.h
-/ext/socket/constdefs.c
-
-# /gems
-/gems/*.gem
-/gems/src
-/gems/*-*
-
-# /lib/
-/lib/ruby/[1-9]*.*
-/lib/ruby/vendor_ruby
-
-# /misc/
-/misc/**/__pycache__
-
-# for `make test-bundler`
-/.rspec_status
-/tool/bundler/*.lock
-
-# /tool/
-/tool/config.guess
-/tool/config.sub
-
-# /win32/
-/win32/*.ico
-
-# RJIT
-/include/ruby-*/*/rb_rjit_min_header-*.h
-/lib/ruby_vm/rjit/instruction.rb
-/lib/ruby_vm/rjit/instruction.rb
-/rjit_config.h
-/rb_rjit_header.h*
-
-# YJIT
-/yjit-bench
-/yjit_exit_locations.dump
-
-# /wasm/
-/wasm/tests/*.wasm
-
-# YARP
-/lib/yarp/mutation_visitor.rb
-/lib/yarp/node.rb
-/lib/yarp/serialize.rb
-/yarp/api_node.c
-/yarp/ast.h
-/yarp/node.c
-/yarp/prettyprint.c
-/yarp/serialize.c
-/yarp/token_type.c

lib/yarp/ffi.rb

@@ -166,6 +166,14 @@ module YARP
         end
       end
     end
+
+    def self.dump_internal(source, source_size, filepath)
+      YPBuffer.with do |buffer|
+        metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
+        yp_parse_serialize(source, source_size, buffer.pointer, metadata)
+        buffer.read
+      end
+    end
   end
 
   # Mark the LibRubyParser module as private as it should only be called through
@@ -175,24 +183,15 @@ module YARP
   # The version constant is set by reading the result of calling yp_version.
   VERSION = LibRubyParser.yp_version.read_string
 
-  def self.dump_internal(source, source_size, filepath)
-    LibRubyParser::YPBuffer.with do |buffer|
-      metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
-      LibRubyParser.yp_parse_serialize(source, source_size, buffer.pointer, metadata)
-      buffer.read
-    end
-  end
-  private_class_method :dump_internal
-
   # Mirror the YARP.dump API by using the serialization API.
   def self.dump(code, filepath = nil)
-    dump_internal(code, code.bytesize, filepath)
+    LibRubyParser.dump_internal(code, code.bytesize, filepath)
   end
 
   # Mirror the YARP.dump_file API by using the serialization API.
   def self.dump_file(filepath)
     LibRubyParser::YPString.with(filepath) do |string|
-      dump_internal(string.source, string.length, filepath)
+      LibRubyParser.dump_internal(string.source, string.length, filepath)
     end
   end
 

yarp/config.yml

@@ -1770,6 +1770,7 @@ nodes:
         type: location
       - name: content_loc
         type: location
+        semantic_field: true # https://github.com/ruby/yarp/issues/1452
       - name: closing_loc
         type: location
       - name: unescaped
@@ -2090,6 +2091,7 @@ nodes:
         type: location
       - name: content_loc
         type: location
+        semantic_field: true # https://github.com/ruby/yarp/issues/1452
       - name: closing_loc
         type: location
       - name: unescaped
@@ -2283,8 +2285,10 @@ nodes:
         kind: StringFlags
       - name: opening_loc
         type: location?
+        semantic_field: true # https://github.com/ruby/yarp/issues/1452
       - name: content_loc
         type: location
+        semantic_field: true # https://github.com/ruby/yarp/issues/1452
       - name: closing_loc
         type: location?
       - name: unescaped

yarp/templates/include/yarp/ast.h.erb

@@ -111,4 +111,6 @@ typedef enum {
 } yp_<%= flag.human %>_t;
 <%- end -%>
 
+#define YP_SERIALIZE_ONLY_SEMANTICS_FIELDS <%= YARP::SERIALIZE_ONLY_SEMANTICS_FIELDS %>
+
 #endif // YARP_AST_H

yarp/templates/lib/yarp/serialize.rb.erb

@@ -89,6 +89,10 @@ module YARP
       def load_nodes
         raise "Invalid serialization" if io.read(4) != "YARP"
         raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION]
+        only_semantic_fields = io.read(1).unpack1("C")
+        unless only_semantic_fields == 0
+          raise "Invalid serialization (location fields must be included but are not)"
+        end
 
         @encoding = load_encoding
         @input = input.force_encoding(@encoding).freeze

yarp/templates/src/serialize.c.erb

@@ -95,14 +95,18 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
                 yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]));
             }
             <%- when YARP::LocationField -%>
+            <%- if field.should_be_serialized? -%>
             yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
+            <%- end -%>
             <%- when YARP::OptionalLocationField -%>
+            <%- if field.should_be_serialized? -%>
             if (((yp_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) {
                 yp_buffer_append_u8(buffer, 0);
             } else {
                 yp_buffer_append_u8(buffer, 1);
                 yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
             }
+            <%- end -%>
             <%- when YARP::UInt32Field -%>
             yp_buffer_append_u32(buffer, ((yp_<%= node.human %>_t *)node)-><%= field.name %>);
             <%- when YARP::FlagsField -%>

yarp/templates/template.rb

@@ -7,6 +7,8 @@ require "yaml"
 module YARP
   COMMON_FLAGS = 2
 
+  SERIALIZE_ONLY_SEMANTICS_FIELDS = ENV.fetch("YARP_SERIALIZE_ONLY_SEMANTICS_FIELDS", false)
+
   # This represents a field on a node. It contains all of the necessary
   # information to template out the code for that field.
   class Field
@@ -15,6 +17,14 @@ module YARP
     def initialize(name:, type:, **options)
       @name, @type, @options = name, type, options
     end
+
+    def semantic_field?
+      true
+    end
+
+    def should_be_serialized?
+      SERIALIZE_ONLY_SEMANTICS_FIELDS ? semantic_field? : true
+    end
   end
 
   # Some node fields can be specialized if they point to a specific kind of
@@ -122,6 +132,10 @@ module YARP
 
   # This represents a field on a node that is a location.
   class LocationField < Field
+    def semantic_field?
+      options[:semantic_field] || false
+    end
+
     def rbs_class
       "Location"
     end
@@ -133,6 +147,10 @@ module YARP
 
   # This represents a field on a node that is a location that is optional.
   class OptionalLocationField < Field
+    def semantic_field?
+      options[:semantic_field] || false
+    end
+
     def rbs_class
       "Location?"
     end
@@ -192,6 +210,10 @@ module YARP
       @comment = config.fetch("comment")
     end
 
+    def semantic_fields
+      @semantic_fields ||= @fields.select(&:semantic_field?)
+    end
+
     # Should emit serialized length of node so implementations can skip
     # the node to enable lazy parsing.
     def needs_serialized_length?

yarp/yarp.c

@@ -14472,6 +14472,7 @@ yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
     yp_buffer_append_u8(buffer, YP_VERSION_MAJOR);
     yp_buffer_append_u8(buffer, YP_VERSION_MINOR);
     yp_buffer_append_u8(buffer, YP_VERSION_PATCH);
+    yp_buffer_append_u8(buffer, YP_SERIALIZE_ONLY_SEMANTICS_FIELDS ? 1 : 0);
 
     yp_serialize_content(parser, node, buffer);
     yp_buffer_append_str(buffer, "\0", 1);