From 70ba3102129dbaf6c943d1ed042103e0238842f4 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 14 Mar 2023 15:38:58 -0700 Subject: [PATCH] YJIT: Introduce no_gc attribute (#7511) --- compile.c | 3 +++ kernel.rb | 2 +- tool/mk_builtin_loader.rb | 2 +- vm_core.h | 4 +++- yjit.c | 29 +++++++++++++++++------------ yjit/bindgen/src/main.rs | 5 +++-- yjit/src/codegen.rs | 22 +++++++++++----------- yjit/src/cruby_bindings.inc.rs | 7 +++++-- 8 files changed, 44 insertions(+), 30 deletions(-) diff --git a/compile.c b/compile.c index 572e727085..8aa2228c53 100644 --- a/compile.c +++ b/compile.c @@ -8236,6 +8236,9 @@ compile_builtin_attr(rb_iseq_t *iseq, const NODE *node) if (strcmp(RSTRING_PTR(string), "leaf") == 0) { ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_LEAF; } + else if (strcmp(RSTRING_PTR(string), "no_gc") == 0) { + ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_NO_GC; + } else { goto unknown_arg; } diff --git a/kernel.rb b/kernel.rb index c8cbc99175..1cd11ab7c5 100644 --- a/kernel.rb +++ b/kernel.rb @@ -16,7 +16,7 @@ module Kernel #++ # def class - Primitive.attr! :leaf + Primitive.attr! :leaf, :no_gc Primitive.cexpr! 'rb_obj_class(self)' end diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb index ba5e3c1ad7..5ab427ca9b 100644 --- a/tool/mk_builtin_loader.rb +++ b/tool/mk_builtin_loader.rb @@ -6,7 +6,7 @@ require_relative 'ruby_vm/helpers/c_escape' SUBLIBS = {} REQUIRED = {} -BUILTIN_ATTRS = %w[leaf] +BUILTIN_ATTRS = %w[leaf no_gc] def string_literal(lit, str = []) while lit diff --git a/vm_core.h b/vm_core.h index ee995b5ff9..66602efea7 100644 --- a/vm_core.h +++ b/vm_core.h @@ -370,8 +370,10 @@ enum rb_iseq_type { // Attributes specified by Primitive.attr! enum rb_builtin_attr { - // If true, this ISeq does not call methods. + // The iseq does not call methods. BUILTIN_ATTR_LEAF = 0x01, + // The iseq does not allocate objects. + BUILTIN_ATTR_NO_GC = 0x02, }; struct rb_iseq_constant_body { diff --git a/yjit.c b/yjit.c index e5431e79b9..5bb8a37e53 100644 --- a/yjit.c +++ b/yjit.c @@ -724,28 +724,33 @@ rb_optimized_call(VALUE *recv, rb_execution_context_t *ec, int argc, VALUE *argv return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler); } +unsigned int +rb_yjit_iseq_builtin_attrs(const rb_iseq_t *iseq) +{ + return iseq->body->builtin_attrs; +} -// If true, the iseq is leaf and it can be replaced by a single C call. -bool -rb_leaf_invokebuiltin_iseq_p(const rb_iseq_t *iseq) +// If true, the iseq has only opt_invokebuiltin_delegate_leave and leave insns. +static bool +invokebuiltin_delegate_leave_p(const rb_iseq_t *iseq) { unsigned int invokebuiltin_len = insn_len(BIN(opt_invokebuiltin_delegate_leave)); unsigned int leave_len = insn_len(BIN(leave)); - - return (iseq->body->iseq_size == (invokebuiltin_len + leave_len) && + return iseq->body->iseq_size == (invokebuiltin_len + leave_len) && rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[0]) == BIN(opt_invokebuiltin_delegate_leave) && - rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave) && - (iseq->body->builtin_attrs & BUILTIN_ATTR_LEAF) != 0 - ); + rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave); } -// Return an rb_builtin_function if the iseq contains only that leaf builtin function. +// Return an rb_builtin_function if the iseq contains only that builtin function. const struct rb_builtin_function * -rb_leaf_builtin_function(const rb_iseq_t *iseq) +rb_yjit_builtin_function(const rb_iseq_t *iseq) { - if (!rb_leaf_invokebuiltin_iseq_p(iseq)) + if (invokebuiltin_delegate_leave_p(iseq)) { + return (const struct rb_builtin_function *)iseq->body->iseq_encoded[1]; + } + else { return NULL; - return (const struct rb_builtin_function *)iseq->body->iseq_encoded[1]; + } } VALUE diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index 031573621f..bf762c1401 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -286,6 +286,7 @@ fn main() { .allowlist_var("VM_ENV_DATA_INDEX_FLAGS") .allowlist_var("VM_ENV_DATA_SIZE") .allowlist_function("rb_iseq_path") + .allowlist_type("rb_builtin_attr") // From yjit.c .allowlist_function("rb_iseq_(get|set)_yjit_payload") @@ -296,8 +297,8 @@ fn main() { .allowlist_function("rb_yjit_mark_executable") .allowlist_function("rb_yjit_mark_unused") .allowlist_function("rb_yjit_get_page_size") - .allowlist_function("rb_leaf_invokebuiltin_iseq_p") - .allowlist_function("rb_leaf_builtin_function") + .allowlist_function("rb_yjit_iseq_builtin_attrs") + .allowlist_function("rb_yjit_builtin_function") .allowlist_function("rb_set_cfp_(pc|sp)") .allowlist_function("rb_cfp_get_iseq") .allowlist_function("rb_yjit_multi_ractor_p") diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 5dae4acf84..db2e371df3 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -5591,14 +5591,10 @@ fn gen_send_iseq( } } - let leaf_builtin_raw = unsafe { rb_leaf_builtin_function(iseq) }; - let leaf_builtin: Option<*const rb_builtin_function> = if leaf_builtin_raw.is_null() { - None - } else { - Some(leaf_builtin_raw) - }; - if let (None, Some(builtin_info)) = (block, leaf_builtin) { - + let builtin_attrs = unsafe { rb_yjit_iseq_builtin_attrs(iseq) }; + let builtin_func_raw = unsafe { rb_yjit_builtin_function(iseq) }; + let builtin_func = if builtin_func_raw.is_null() { None } else { Some(builtin_func_raw) }; + if let (None, Some(builtin_info), true) = (block, builtin_func, builtin_attrs & BUILTIN_ATTR_LEAF != 0) { // this is a .send call not currently supported for builtins if flags & VM_CALL_OPT_SEND != 0 { gen_counter_incr!(asm, send_send_builtin); @@ -5609,9 +5605,13 @@ fn gen_send_iseq( if builtin_argc + 1 < (C_ARG_OPNDS.len() as i32) { asm.comment("inlined leaf builtin"); - // Save the PC and SP because the callee may allocate - // e.g. Integer#abs on a bignum - jit_prepare_routine_call(jit, ctx, asm); + // Skip this if it doesn't trigger GC + if builtin_attrs & BUILTIN_ATTR_NO_GC == 0 { + // The callee may allocate, e.g. Integer#abs on a Bignum. + // Save SP for GC, save PC for allocation tracing, and prepare + // for global invalidation after GC's VM lock contention. + jit_prepare_routine_call(jit, ctx, asm); + } // Call the builtin func (ec, recv, arg1, arg2, ...) let mut args = vec![EC]; diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 18c519f325..190b163b41 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -662,6 +662,9 @@ pub struct iseq_inline_iv_cache_entry { pub struct iseq_inline_cvar_cache_entry { pub entry: *mut rb_cvar_class_tbl_entry, } +pub const BUILTIN_ATTR_LEAF: rb_builtin_attr = 1; +pub const BUILTIN_ATTR_NO_GC: rb_builtin_attr = 2; +pub type rb_builtin_attr = u32; #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword { @@ -1276,8 +1279,8 @@ extern "C" { kw_splat: ::std::os::raw::c_int, block_handler: VALUE, ) -> VALUE; - pub fn rb_leaf_invokebuiltin_iseq_p(iseq: *const rb_iseq_t) -> bool; - pub fn rb_leaf_builtin_function(iseq: *const rb_iseq_t) -> *const rb_builtin_function; + pub fn rb_yjit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; + pub fn rb_yjit_builtin_function(iseq: *const rb_iseq_t) -> *const rb_builtin_function; pub fn rb_yjit_str_simple_append(str1: VALUE, str2: VALUE) -> VALUE; pub fn rb_get_ec_cfp(ec: *const rb_execution_context_t) -> *mut rb_control_frame_struct; pub fn rb_get_cfp_pc(cfp: *mut rb_control_frame_struct) -> *mut VALUE;